class_docstr: This tokenizer is based on GPTNeoXJapaneseTokenizer and has the following modifications
  - Decoding byte0~byte255 tokens correctly
  - Added bagofword token handling
  
  https://github.com/tanreinama/Japanese-BPEEncoder_V2 This tokenizer class is under MIT Lisence according to the
  original repository.
  
  MIT License
  
  Copyright (c) 2020 tanreinama
  
  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
  documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to the following conditions:
  
  The above copyright notice and this permission notice shall be included in all copies or substantial portions of
  the Software.
  
  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
  THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  SOFTWARE.
class_docstr_tok: ["This","tokenizer","is","based","on","GPTNeoXJapaneseTokenizer","and","has","the","following","modifications","-","Decoding","byte0~byte255","tokens","correctly","-","Added","bagofword","token","handling","https",":","\/\/github.com\/tanreinama\/Japanese-BPEEncoder_V2","This","tokenizer","class","is","under","MIT","Lisence","according","to","the","original","repository",".","MIT","License","Copyright","(","c",")","2020","tanreinama","Permission","is","hereby","granted",",","free","of","charge",",","to","any","person","obtaining","a","copy","of","this","software","and","associated","documentation","files","(","the","``","Software","''",")",",","to","deal","in","the","Software","without","restriction",",","including","without","limitation","the","rights","to","use",",","copy",",","modify",",","merge",",","publish",",","distribute",",","sublicense",",","and\/or","sell","copies","of","the","Software",",","and","to","permit","persons","to","whom","the","Software","is","furnished","to","do","so",",","subject","to","the","following","conditions",":","The","above","copyright","notice","and","this","permission","notice","shall","be","included","in","all","copies","or","substantial","portions","of","the","Software",".","THE","SOFTWARE","IS","PROVIDED","``","AS","IS","''",",","WITHOUT","WARRANTY","OF","ANY","KIND",",","EXPRESS","OR","IMPLIED",",","INCLUDING","BUT","NOT","LIMITED","TO","THE","WARRANTIES","OF","MERCHANTABILITY",",","FITNESS","FOR","A","PARTICULAR","PURPOSE","AND","NONINFRINGEMENT",".","IN","NO","EVENT","SHALL","THE","AUTHORS","OR","COPYRIGHT","HOLDERS","BE","LIABLE","FOR","ANY","CLAIM",",","DAMAGES","OR","OTHER","LIABILITY",",","WHETHER","IN","AN","ACTION","OF","CONTRACT",",","TORT","OR","OTHERWISE",",","ARISING","FROM",",","OUT","OF","OR","IN","CONNECTION","WITH","THE","SOFTWARE","OR","THE","USE","OR","OTHER","DEALINGS","IN","THE","SOFTWARE","."]
code: def __init__(self, vocab, ids_to_tokens, emoji):
      self.vocab = vocab  # same as swe
      self.ids_to_tokens = ids_to_tokens  # same as bpe
      self.emoji = emoji
      self.maxlen = np.max([len(w) for w in self.vocab.keys()])
      self.content_repatter1 = re.compile(
          r"(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"
      )
      self.content_repatter2 = re.compile(
          r"[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"
      )
      self.content_repatter3 = re.compile(
          r"[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"
      )
      self.content_repatter4 = re.compile(
          r"([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"
      )
      self.content_repatter5 = re.compile(
          r"(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"
      )
      self.content_repatter6 = re.compile(
          r"((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"
      )
      keisen = "─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"
      blocks = "▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"
      self.content_trans1 = str.maketrans(
          {k: "<BLOCK>" for k in keisen + blocks}
      )
  
imports: import collections
  import json
  import os
  import re
  from typing import List, Optional, Tuple, Union
  import numpy
  from ...tokenization_utils import PreTrainedTokenizer
  from ...tokenization_utils_base import BatchEncoding, PreTokenizedInput, PreTokenizedInputPair, TextInput, TextInputPair, TruncationStrategy
  from ...utils import PaddingStrategy, logging
label_desc: Use image node_id 1 to create a new SubWordJapaneseTokenizer object from inherited base classes: object with example: obj = SubWordJapaneseTokenizer(vocab, ids_to_tokens, emoji)