class_docstr: Object required by Translator.translate().
  If not None, `pass_through_dict` is an arbitrary dictionary instantiated from a JSON object
  via `make_input_from_dict()`, and it contains extra fields found in an input JSON object.
  If `--output-type json` is selected, all such fields that are not fields used or changed by
  Sockeye will be included in the output JSON object. This provides a mechanism for passing
  fields through the call to Sockeye.
class_docstr_tok: ["Object","required","by","Translator.translate","(",")",".","If","not","None",",","`","pass_through_dict","`","is","an","arbitrary","dictionary","instantiated","from","a","JSON","object","via","`","make_input_from_dict","(",")","`",",","and","it","contains","extra","fields","found","in","an","input","JSON","object",".","If","`","--","output-type","json","`","is","selected",",","all","such","fields","that","are","not","fields","used","or","changed","by","Sockeye","will","be","included","in","the","output","JSON","object",".","This","provides","a","mechanism","for","passing","fields","through","the","call","to","Sockeye","."]
docstr: Takes a TranslatorInput (itself) and yields TranslatorInputs for chunks of size chunk_size.
  
  :param chunk_size: The maximum size of a chunk.
  :return: A generator of TranslatorInputs, one for each chunk created.
docstr_tok: ["Takes","a","TranslatorInput","(","itself",")","and","yields","TranslatorInputs","for","chunks","of","size","chunk_size",".",":","param","chunk_size",":","The","maximum","size","of","a","chunk",".",":","return",":","A","generator","of","TranslatorInputs",",","one","for","each","chunk","created","."]
code: def chunks(
      self, chunk_size: int
  ) -> Generator["TranslatorInput", None, None]:
      """
      Takes a TranslatorInput (itself) and yields TranslatorInputs for chunks of size chunk_size.
  
      :param chunk_size: The maximum size of a chunk.
      :return: A generator of TranslatorInputs, one for each chunk created.
      """
  
      if len(self.tokens) > chunk_size and self.constraints is not None:
          logger.warning(
              "Input %s has length (%d) that exceeds max input length (%d), "
              "triggering internal splitting. Placing all target-side constraints "
              "with the first chunk, which is probably wrong.",
              self.sentence_id,
              len(self.tokens),
              chunk_size,
          )
  
      for chunk_id, i in enumerate(
          range(
              0, len(self) - self.num_source_prefix_tokens, chunk_size
          )
      ):
          factors = (
              [factor[i : i + chunk_size] for factor in self.factors]
              if self.factors is not None
              else None
          )
          # Constrained decoding is not supported for chunked TranslatorInputs. As a fall-back, constraints are
          # assigned to the first chunk
          constraints = self.constraints if chunk_id == 0 else None
          # Target_prefix_tokens are assigned to all chunks if self.use_target_prefix_all_chunks is True,
          # otherwise target_prefix_tokens are assigned only to the first chunk
          target_prefix_tokens = (
              self.target_prefix_tokens
              if chunk_id == 0 or self.use_target_prefix_all_chunks
              else None
          )
          target_prefix_factors = (
              self.target_prefix_factors
              if chunk_id == 0 or self.use_target_prefix_all_chunks
              else None
          )
          pass_through_dict = (
              copy.deepcopy(self.pass_through_dict)
              if (chunk_id == 0 and self.pass_through_dict is not None)
              else None
          )
          yield TranslatorInput(
              sentence_id=self.sentence_id,
              tokens=self.tokens[i : i + chunk_size],
              factors=factors,
              source_prefix_tokens=self.source_prefix_tokens,
              source_prefix_factors=self.source_prefix_factors,
              target_prefix_tokens=target_prefix_tokens,
              target_prefix_factors=self.target_prefix_factors,
              use_target_prefix_all_chunks=self.use_target_prefix_all_chunks,
              keep_target_prefix_key=self.keep_target_prefix_key,
              restrict_lexicon=self.restrict_lexicon,
              constraints=constraints,
              avoid_list=self.avoid_list,
              pass_through_dict=pass_through_dict,
          )
  
imports: import copy
  import itertools
  import json
  import logging
  from dataclasses import dataclass
  from functools import partial
  from typing import Any, Callable, Dict, Generator, List, Optional, Set, Tuple, Union
  import numpy
  import torch
  from .None import constants
  from .None import lexicon
  from .None import utils
  from .None import vocab
  from .beam_search import CandidateScorer, get_search_algorithm, GreedySearch, SearchResult
  from .data_io import tokens2ids, get_prepended_token_length
  from .model import SockeyeModel
label_desc: Use image node_id 10 for calling the TranslatorInput obj's underlying member method code with example usage: obj.chunks(chunk_size) without return types