code: def tab_sample_sequence_batch(
      model,
      inference_strategy,
      context_tokens,
      context_lengths,
      tokens_to_generate,
      all_probs=True,
      compute_attention_mask=True,
      type_ids=None,
      temperature=None,
  ):
      app_state = AppState()
      micro_batch_size = context_tokens.shape[0]
      _reconfigure_microbatch_calculator(
          rank=app_state.global_rank,
          rampup_batch_size=None,
          global_batch_size=micro_batch_size,
          micro_batch_size=micro_batch_size,
          data_parallel_size=1,
      )
      tokenizer = model.tokenizer
      sizes = tokenizer.code_column.sizes
      tokens_per_row = sum(sizes) + 1
      columns = tokenizer.code_column.columns
      num_columns = len(columns)
      tokenid_range = []
      for i in range(num_columns):
          tokenid_range.extend(tokenizer.code_column.get_range(i))
      # initialize the batch
      with torch.no_grad():
          context_length = context_lengths.min().item()
          inference_strategy.init_batch(
              context_tokens, context_length, compute_attention_mask
          )
          context = context_tokens[:, :context_length]
          # the context may start in the middle of the row,
          # calculate the offset according to the position of '\n' or '<|endoftext|>'
          positions = torch.where(context == tokenizer.eor)[1]
          if len(positions) == 0:
              positions = torch.where(context == tokenizer.eod)[1]
          if len(positions) != 0:
              max_position = positions.max().item()
              # TODO, need to make sure context of different batch have the same offset lengths")
              # otherwise, need to calculate offset per batch_id
              offset = (
                  context_length - max_position - 1
              ) % tokens_per_row
          else:
              offset = 0
  
          eod_id = tokenizer.eos_id
  
          counter = 0
  
          batch_size = context_tokens.size(0)
          is_done = torch.zeros([batch_size]).byte().cuda()
          tokens = context_tokens
          output_logits = None
  
          # Generate enough tokens for the longest sequence
          maxlen = tokens_to_generate + context_lengths.max().item()
  
          if maxlen > model.cfg.encoder_seq_length:
              maxlen = model.cfg.encoder_seq_length
  
          lengths = torch.ones([batch_size]).long().cuda() * maxlen
  
          while context_length < maxlen:
              (
                  batch,
                  tensor_shape,
              ) = inference_strategy.prepare_batch_at_step(
                  tokens,
                  maxlen,
                  micro_batch_size,
                  counter,
                  context_length,
                  compute_attention_mask,
              )
              output = inference_strategy.forward_step(
                  batch, tensor_shape
              )
  
              if parallel_state.is_pipeline_last_stage():
                  output = output[0]["logits"].float()
                  output = tensor_parallel.gather_from_tensor_model_parallel_region(
                      output
                  )
                  assert output is not None
                  output = output.float()
                  logits = (
                      output[:, -1].view(batch_size, -1).contiguous()
                  )
                  token_in_row = (counter + offset) % tokens_per_row
                  logits = logits.float()
                  logits /= temperature
                  if token_in_row == tokens_per_row - 1:
                      # line break
                      eor_id = tokenizer.eor
                      eod_id = tokenizer.eos_id
                      min_id = min(eor_id, eod_id)
                      max_id = max(eor_id, eod_id) + 1
                      logits = tab_logits(logits, min_id, max_id)
                  else:
                      # limit the range
                      min_id, max_id = tokenid_range[token_in_row]
                      logits = tab_logits(logits, min_id, max_id)
                  probs = F.softmax(logits, dim=-1)
                  prev = torch.multinomial(probs, num_samples=1).view(
                      -1
                  )
                  started = context_lengths <= context_length
                  # Clamp the out of vocabulary tokens.
                  prev = torch.clamp(prev, max=tokenizer.vocab_size - 1)
  
                  new_tokens = switch(
                      tokens[:, context_length].view(-1), prev, started
                  )
  
                  # post process the inference tokens based on the strategy
                  inference_strategy.post_process(
                      tokens, new_tokens, context_length
                  )
  
                  tokens[:, context_length] = new_tokens
  
                  if output_logits is None:
                      output_context = F.log_softmax(
                          output[:, :context_length, :], 2
                      )
                      indices = torch.unsqueeze(
                          tokens[:, 1 : context_length + 1], 2
                      )
                      output_logits = torch.gather(
                          output_context, 2, indices
                      ).squeeze(2)
                      if all_probs:
                          full_logits = output_context
                  else:
                      output_context = F.log_softmax(output, 2)
                      indices = torch.unsqueeze(
                          new_tokens, 1
                      ).unsqueeze(2)
                      new_output_logits = torch.gather(
                          output_context, 2, indices
                      ).squeeze(2)
  
                      # TODO(rprenger) we're copying output_logits every time.  Should pre-allocate
                      output_logits = torch.cat(
                          [output_logits, new_output_logits], 1
                      )
                      if all_probs:
                          full_logits = torch.cat(
                              [full_logits, output_context], 1
                          )
  
                  src = (
                      parallel_state.get_pipeline_model_parallel_last_rank()
                  )
                  group = parallel_state.get_embedding_group()
                  torch.distributed.broadcast(new_tokens, src, group)
  
                  done_token = (prev == eod_id).byte() & started.byte()
                  just_finished = (done_token & ~is_done).bool()
                  lengths[just_finished.view(-1)] = context_length
                  is_done = is_done | done_token
  
                  done = torch.all(is_done)
                  src = (
                      parallel_state.get_pipeline_model_parallel_last_rank()
                  )
                  group = (
                      parallel_state.get_pipeline_model_parallel_group()
                  )
                  torch.distributed.broadcast(done, src, group)
                  if all_probs:
                      yield tokens, lengths, output_logits, full_logits
                  else:
                      yield tokens, lengths, output_logits, None
  
              else:
                  if parallel_state.is_pipeline_first_stage():
                      src = (
                          parallel_state.get_pipeline_model_parallel_last_rank()
                      )
                      group = parallel_state.get_embedding_group()
                      new_tokens = torch.empty_like(
                          tokens[:, context_length]
                      )
                      torch.distributed.broadcast(
                          new_tokens, src, group
                      )
                      tokens[:, context_length] = new_tokens
                      yield tokens, None, None, None
                  else:
                      yield None, None, None, None
  
                  done = torch.cuda.ByteTensor([0])
                  src = (
                      parallel_state.get_pipeline_model_parallel_last_rank()
                  )
                  group = (
                      parallel_state.get_pipeline_model_parallel_group()
                  )
                  torch.distributed.broadcast(done, src, group)
  
              context_length += 1
              counter += 1
              if done:
                  break
  
imports: import os
  import pickle
  import re
  from collections.abc import Iterable
  from functools import partial
  from typing import Callable, Tuple
  import numpy
  import torch
  import torch.nn.functional
  from nemo.collections.common.tokenizers.tabular_tokenizer import TabularTokenizer
  from nemo.collections.nlp.modules.common.megatron.utils import get_ltor_masks_and_position_ids
  from nemo.collections.nlp.modules.common.text_generation_strategy import model_inference_strategy_dispatcher
  from nemo.collections.nlp.modules.common.transformer.text_generation import LengthParam, OutputType, SamplingParam
  from nemo.utils import AppState
label_desc: Use image node_id 17 for calling a global function with example usage: tab_sample_sequence_batch(model, inference_strategy, context_tokens, context_lengths, tokens_to_generate, all_probs, compute_attention_mask, type_ids, temperature) without return types