code: def _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS):
      """Given a set of prompts and number of tokens to generate:
      - tokenize prompts
      - set the sequence length to be the max of length of prompts
        plus the number of tokens we would like to generate
      - pad all the sequences to this length so we can convert them
        into a 2D tensor.
      """
  
      # Tokenize all the prompts.
      tokenizer = get_tokenizer()
      if add_BOS:
          prompts_tokens = [
              [tokenizer.eod] + tokenizer.tokenize(prompt)
              for prompt in prompts
          ]
      else:
          prompts_tokens = [
              tokenizer.tokenize(prompt) for prompt in prompts
          ]
  
      # Now we have a list of list of tokens which each list has a different
      # size. We want to extend this list to:
      #   - incorporate the tokens that need to be generated
      #   - make all the sequences equal length.
      # Get the prompts length.
      prompts_length = [
          len(prompt_tokens) for prompt_tokens in prompts_tokens
      ]
      # Get the max prompts length.
      max_prompt_len = max(prompts_length)
      # Number of tokens in the each sample of the batch.
      samples_length = max_prompt_len + tokens_to_generate
      # Now update the list of list to be of the same size: samples_length.
      for prompt_tokens, prompt_length in zip(
          prompts_tokens, prompts_length
      ):
          padding_size = samples_length - prompt_length
          prompt_tokens.extend([tokenizer.eod] * padding_size)
  
      # Now we are in a structured format, we can convert to tensors.
      prompts_tokens_tensor = torch.tensor(
          prompts_tokens, dtype=torch.long, device="cuda"
      )
      prompts_length_tensor = torch.tensor(
          prompts_length, dtype=torch.long, device="cuda"
      )
  
      return prompts_tokens_tensor, prompts_length_tensor
  
imports: import torch
  from megatron import get_tokenizer, get_args
  from .communication import broadcast_int_list, broadcast_tensor
label_desc: Use image node_id 3 for calling a global function with example usage: _tokenize_prompts_and_batch(prompts, tokens_to_generate, add_BOS) and returns: prompts_tokens_tensor,  prompts_length_tensor