class_docstr: Test class for prompt tokenization strategies.
class_docstr_tok: ["Test","class","for","prompt","tokenization","strategies","."]
docstr: Make sure the sharegpt/llama is tokenized and formatted correctly.
docstr_tok: ["Make","sure","the","sharegpt\/llama","is","tokenized","and","formatted","correctly","."]
code: def test_sharegpt_llama(self):
      "Make sure the sharegpt/llama is tokenized and formatted correctly."
      strat = prompt_strat("llama-2", self.tokenizer)
  
      def tokenize(conv):
          return strat.tokenize_prompt(deepcopy(conv))["input_ids"]
  
      def decode(ids):
          return strat.tokenizer.decode(ids)
  
          # fmt: off
          # System message, multi-turn conversations
          mt_ids = tokenize(test_data['multi_turn_sys'])
          assert decode(mt_ids) == '<s> [INST] <<SYS>>\nlorem\n<</SYS>>\n\nabc [/INST] ipsum</s><s> [INST] 123 [/INST] sit</s>'
          assert mt_ids == [1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 29880, 3668, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 10736, 518, 29914, 25580, 29962, 23421, 2, 1, 518, 25580, 29962, 29871, 29896, 29906, 29941, 518, 29914, 25580, 29962, 7845, 2]
  
          # System message, single-turn conversations
          st_ids = tokenize(test_data['single_turn_sys'])
          assert decode(st_ids) == '<s> [INST] <<SYS>>\nlorem\n<</SYS>>\n\nabc [/INST] ipsum</s>'
          assert st_ids == [1, 518, 25580, 29962, 3532, 14816, 29903, 6778, 13, 29880, 3668, 13, 29966, 829, 14816, 29903, 6778, 13, 13, 10736, 518, 29914, 25580, 29962, 23421, 2]
  
          # No system message, single-turn
          ns_ids = tokenize(test_data['single_turn_no_sys'])
          assert decode(ns_ids) == '<s> [INST] abc [/INST] ipsum</s>'
          assert ns_ids == [1, 518, 25580, 29962, 25638, 518, 29914, 25580, 29962, 23421, 2]
  
          # No system message, multi-turn
          ns_mt_ids = tokenize(test_data['multi_turn_no_sys'])
          assert decode(ns_mt_ids) == '<s> [INST] abc [/INST] ipsum</s><s> [INST] 123 [/INST] sit</s>'
          assert ns_mt_ids == [1, 518, 25580, 29962, 25638, 518, 29914, 25580, 29962, 23421, 2, 1, 518, 25580, 29962, 29871, 29896, 29906, 29941, 518, 29914, 25580, 29962, 7845, 2]
  
imports: import json
  import logging
  import unittest
  from copy import deepcopy
  from pathlib import Path
  from typing import Optional
  import pytest
  from transformers import AutoTokenizer, LlamaTokenizer
  from axolotl.prompt_strategies.alpaca_chat import NoSystemPrompter
  from axolotl.prompt_strategies.alpaca_w_system import InstructionWSystemPromptTokenizingStrategy, SystemDataPrompter
  from axolotl.prompt_strategies.llama2_chat import Llama2ChatPrompter, LLama2ChatTokenizingStrategy
  from axolotl.prompt_tokenizers import AlpacaPromptTokenizingStrategy, ShareGPTPromptTokenizingStrategy
  from axolotl.prompters import AlpacaPrompter, PromptStyle, ShareGPTPrompterV2
label_desc: Use image node_id 6 for calling the TestPromptTokenizationStrategies obj's underlying member method code with example usage: obj.test_sharegpt_llama() and returns: strat, strat