code: def tokenize_prompts(
      prompts=None, tokens_to_generate=None, add_BOS=None, rank=0
  ):
      """Tokenize prompts and make them avaiable on all ranks."""
  
      # On all ranks set to None so we can pass them to functions
      sizes_list = None
      prompts_tokens_cuda_long_tensor = None
      prompts_length_cuda_long_tensor = None
  
      # On the specified rank, build the above.
      if torch.distributed.get_rank() == rank:
          assert prompts is not None
          assert tokens_to_generate is not None
          # Tensor of tokens padded and their unpadded length.
          (
              prompts_tokens_cuda_long_tensor,
              prompts_length_cuda_long_tensor,
          ) = _tokenize_prompts_and_batch(
              prompts, tokens_to_generate, add_BOS
          )
          # We need the sizes of these tensors for the boradcast
          sizes_list = [
              prompts_tokens_cuda_long_tensor.size(0),  # Batch size
              prompts_tokens_cuda_long_tensor.size(1),
          ]  # Sequence lenght
  
      # First, broadcast the sizes.
      sizes_tensor = broadcast_int_list(
          2, int_list=sizes_list, rank=rank
      )
  
      # Now that we have the sizes, we can boradcast the tokens
      # and length tensors.
      sizes = sizes_tensor.tolist()
      prompts_tokens_cuda_long_tensor = broadcast_tensor(
          sizes,
          torch.int64,
          tensor=prompts_tokens_cuda_long_tensor,
          rank=rank,
      )
      prompts_length_cuda_long_tensor = broadcast_tensor(
          sizes[0],
          torch.int64,
          tensor=prompts_length_cuda_long_tensor,
          rank=rank,
      )
  
      return (
          prompts_tokens_cuda_long_tensor,
          prompts_length_cuda_long_tensor,
      )
  
imports: import torch
  from megatron import get_tokenizer, get_args
  from .communication import broadcast_int_list, broadcast_tensor
label_desc: Use image node_id 2 for calling a global function with example usage: tokenize_prompts(prompts, tokens_to_generate, add_BOS, rank) and returns: prompts_tokens_cuda_long_tensor,  prompts_length_cuda_long_tensor