class_docstr: A class that regroups important test to make sure that we properly handle the special tokens.
class_docstr_tok: ["A","class","that","regroups","important","test","to","make","sure","that","we","properly","handle","the","special","tokens","."]
code: def test_remove_extra_whitespaces(self):
      # make sure the extra spaces are eaten
      # sentencepiece.NormalizerSpec.remove_extra_whitespaces attribute
      input_ids = self.tokenizer.encode(
          "       . Hello", add_special_tokens=False
      )
      self.assertEqual(input_ids, [37, 86, 20])
      self.assertEqual(input_ids, [37, 86, 20])
      tokens = self.tokenizer.tokenize(" . Hello")
      self.assertEqual(tokens, ["▁he", "ll", "o"])
  
      # `'▁'` is also a whitespace
      input_ids = self.tokenizer.encode("▁He is not")
      self.assertEqual(input_ids, [37, 46, 44, 2])
      tokens = self.tokenizer.tokenize("▁He is not")
      self.assertEqual(
          tokens, ["▁he", "▁is", "▁not"]
      )  # no extra space added
  
      input_ids = self.tokenizer.encode("▁He is not             ▁He")
      self.assertEqual(input_ids, [37, 46, 44, 37, 2])
      tokens = self.tokenizer.tokenize("▁He is not              ▁He")
      self.assertEqual(tokens, ["▁he", "▁is", "▁not", "▁he"])
  
imports: import json
  import os
  import tempfile
  import unittest
  from transformers import SPIECE_UNDERLINE, AddedToken, BatchEncoding, SiglipTokenizer
  from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
  from transformers.utils import cached_property, is_tf_available, is_torch_available
  from ...test_tokenization_common import TokenizerTesterMixin
label_desc: Use image node_id 3 for calling the CommonSpmIntegrationTests obj's underlying member method code with example usage: obj.test_remove_extra_whitespaces() without return types