docstr: Args:
      gpus (int or [int]): list of GPU ids.
      average (bool): whether to average or sum gradients.
      mode (str or None): Gradient aggregation mode.
          Supported values: ['nccl', 'hierarchical', 'cpu', 'gpu'].
          These modes may differ in speed.
          Default to pick automatically by heuristics.
          "hierarchical" mode was designed for DGX-like 8-GPU machines.
docstr_tok: ["Args",":","gpus","(","int","or","[","int","]",")",":","list","of","GPU","ids",".","average","(","bool",")",":","whether","to","average","or","sum","gradients",".","mode","(","str","or","None",")",":","Gradient","aggregation","mode",".","Supported","values",":","[","'nccl","'",",","'hierarchical","'",",","'cpu","'",",","'gpu","'","]",".","These","modes","may","differ","in","speed",".","Default","to","pick","automatically","by","heuristics",".","``","hierarchical","''","mode","was","designed","for","DGX-like","8-GPU","machines","."]
code: def __init__(self, gpus, average=True, mode=None):
      """
      Args:
          gpus (int or [int]): list of GPU ids.
          average (bool): whether to average or sum gradients.
          mode (str or None): Gradient aggregation mode.
              Supported values: ['nccl', 'hierarchical', 'cpu', 'gpu'].
              These modes may differ in speed.
              Default to pick automatically by heuristics.
              "hierarchical" mode was designed for DGX-like 8-GPU machines.
      """
      self.devices = gpus
      if mode is not None:
          mode = mode.lower()
  
      # Heuristics about mode selection:
      if mode == "hierarchical" and len(gpus) != 8:
          logger.warn(
              "mode='hierarchical' requires 8 GPUs. Will fallback to default mode."
          )
          mode = None
      if mode is None:
          if len(gpus) == 8:
              mode = "hierarchical"
          else:
              # https://github.com/tensorflow/tensorflow/issues/41539
              mode = (
                  "nccl" if get_tf_version_tuple() < (1, 15) else "gpu"
              )
      if mode == "cpu" and get_tf_version_tuple() >= (2, 0):
          # cpu mode causes the entire model to get located on cpu
          mode = "gpu"
      if mode == "nccl" and get_tf_version_tuple() >= (1, 15):
          logger.warning(
              "NCCL in TensorFlow has a serious bug that is likely to trigger in TF>=1.15. "
              "Try 'mode=None' to use a better default mode."
          )
  
      self._builder = SyncMultiGPUReplicatedBuilder(gpus, average, mode)
      self.BROADCAST_EVERY_EPOCH = True
  
      super(SyncMultiGPUTrainerReplicated, self).__init__()
  
imports: import multiprocessing
  import os
  import sys
  import tensorflow
  from tensorpack.compat import tfv1
  from ..callbacks import CallbackFactory, RunOp
  from ..graph_builder.distributed import DistributedParameterServerBuilder, DistributedReplicatedBuilder
  from ..graph_builder.training import AsyncMultiGPUBuilder, SyncMultiGPUParameterServerBuilder, SyncMultiGPUReplicatedBuilder
  from ..graph_builder.utils import override_to_local_variable
  from ..input_source import FeedfreeInput, QueueInput
  from ..tfutils import get_global_step_var
  from ..tfutils.common import get_tf_version_tuple
  from ..tfutils.distributed import get_distributed_session_creator
  from ..tfutils.sesscreate import NewSessionCreator
  from ..tfutils.tower import TrainTowerContext
  from ..utils import logger
  from ..utils.argtools import map_arg
  from ..utils.develop import HIDE_DOC, deprecated
  from .tower import SingleCostTrainer
label_desc: Use image node_id 1 to create a new SyncMultiGPUTrainerReplicated object from inherited base classes: SingleCostTrainer with example: obj = SyncMultiGPUTrainerReplicated(gpus, average, mode)