class_docstr: PipelineHelper assists with implementing pipelining in model parallelism.
  
  For a tutorial on model parallelism, as it's implemented in parts of ParlAI,
  see https://pytorch.org/tutorials/intermediate/model_parallel_tutorial.html.
  
  Usage:
  >>> my_model = PipelineHelper().make_parallel(my_model)
  
  Note that you will need to manually implement logic which handles the
  moved layers.
class_docstr_tok: ["PipelineHelper","assists","with","implementing","pipelining","in","model","parallelism",".","For","a","tutorial","on","model","parallelism",",","as","it","'s","implemented","in","parts","of","ParlAI",",","see","https",":","\/\/pytorch.org\/tutorials\/intermediate\/model_parallel_tutorial.html",".","Usage",":",">",">",">","my_model","=","PipelineHelper","(",")",".make_parallel","(","my_model",")","Note","that","you","will","need","to","manually","implement","logic","which","handles","the","moved","layers","."]
code: def _place_modulelist(self, submodule: torch.nn.Module) -> None:
      if not isinstance(submodule, torch.nn.ModuleList):
          # not a ModuleList, leave it untouched
          return
      if getattr(submodule, "model_parallel_exempt", False):
          return
  
      assert isinstance(
          submodule, torch.nn.ModuleList
      )  # for typechecker
      layers = submodule
  
      # mark this section as MP
      layers.is_model_parallel = True  # type: ignore
  
      # next, let's figure out how many parameters we can assign to each GPU,
      # but not make actual assignments yet. Assignments come later because we
      # want consecutive layers to be collocated
      keyfunc = self.__device_allocations.__getitem__
      layer_assignments = {k: 0 for k in self.devices}
      for layer_no, layer in enumerate(layers):
          if layer_no == 0:
              # hard code the first layer to be 0.
              mostfree = "cuda:0"
          else:
              # otherwise dynamic allocation
              mostfree = min(self.devices, key=keyfunc)
          # 32 is a totally arbitrary, made up number that worked in practice
          # on the large models I tested on. I believe it should be roughly
          # batch size, but this was set empirically.
          self.__device_allocations[mostfree] += (
              trainable_parameters(layer) * 32
          )
          # mark a layer as going to the given element
          layer_assignments[mostfree] += 1
  
      devices = [
          d
          for i, d in enumerate(self.devices[:])
          if layer_assignments[d] > 0
      ]
      for layer_no, layer in enumerate(layers):
          layer_gpu = devices[0]
          assert layer_assignments[layer_gpu] > 0
          logging.debug(
              f"Model Parallel: Assigning {layer_no} to {layer_gpu}"
          )
          layer._mp_gpu = layer_gpu
          layers[layer_no] = layer.to(layer_gpu)
          layer_assignments[layer_gpu] -= 1
          if layer_assignments[layer_gpu] == 0:
              devices.pop(0)
  
imports: import os
  from typing import Union, Optional, Tuple, Any, List, Sized, TypeVar
  import itertools
  from collections import namedtuple
  import parlai.utils.logging
  import parlai.utils.io
  import torch.optim
label_desc: Use image node_id 5 for calling the PipelineHelper obj's underlying member method code with example usage: obj._place_modulelist(submodule) without return types