class_docstr: Implements incremental block distributed data parallelism similar to
  https://ieeexplore.ieee.org/document/7472805
  
  Paper title: Scalable training of deep learning machines by incremental
  block training with intra-block parallel optimization and blockwise
  model-update filtering
class_docstr_tok: ["Implements","incremental","block","distributed","data","parallelism","similar","to","https",":","\/\/ieeexplore.ieee.org\/document\/7472805","Paper","title",":","Scalable","training","of","deep","learning","machines","by","incremental","block","training","with","intra-block","parallel","optimization","and","blockwise","model-update","filtering"]
code: def __init__(self, cfg: FairseqBMUFConfig, optimizer):
      super().__init__(cfg)
      self._optimizer = optimizer
      self._num_updates = 0
      self.sync_iter = cfg.global_sync_iter
      self.block_momentum = cfg.block_momentum
      self.block_lr = cfg.block_lr
      self._reset_local_data()
      self.warmup_iteration = cfg.warmup_iterations
      self.use_nbm = cfg.use_nbm
      self.initial_state = self._optimizer.state_dict()
      self.average_sync = self.cfg.average_sync
      self.world_size = self.cfg.distributed_world_size
  
imports: from dataclasses import dataclass, field
  import torch
  import torch.distributed
  from fairseq.dataclass.configs import FairseqBMUFConfig
  from fairseq.dataclass.utils import gen_parser_from_dataclass
  from fairseq.optim.fairseq_optimizer import FairseqOptimizer
label_desc: Use image node_id 1 to create a new FairseqBMUF object from inherited base classes: FairseqOptimizer with example: obj = FairseqBMUF(cfg, optimizer)