code: def _bias_dropout_add_func(x_with_bias, residual, prob, training):
      # type: (Tuple[Tensor, Optional[Tensor]], Tensor, float, bool) -> Tensor
      # NOTE: Previously, the argument `bias` used to be passed as
      # `bias.expand_as(residual)` when the `bias_dropout_func` is called from the
      # transformer layer but broadcasting should automatically take care of that.
      # Also, looking at broadcasting semantics, `expand_as` and broadcasting
      # seem to be identical performance-wise (both just change the view).
  
      x, bias = x_with_bias  # unpack
  
      # If we want to train mixed precision, then the output of this function
      # should be half precision. However, in AMP O1, the input (residual) is
      # in fp32, and it will up-cast the result to fp32, causing pipeline parallel
      # GPU communication to hang. Therefore, we need to cast residual to the same
      # dtype as x.
      residual = (
          residual
          if residual.dtype == x.dtype
          else residual.to(x.dtype)
      )
  
      # The Dropout operation, Residual Addition and the tensor returning can be
      # done generically outside the if statement, but that stops fusing of Bias
      # Addition-Dropout-Residual Addition operation. So doing it together inside
      # the conditional branch to improve performance
      if bias is not None:
          x = x + bias
          out = torch.nn.functional.dropout(
              x, p=prob, training=training
          )
          out = residual + out
          return out
      else:
          out = torch.nn.functional.dropout(
              x, p=prob, training=training
          )
          out = residual + out
          return out
  
imports: from typing import Optional, Tuple
  import torch
label_desc: Use image node_id 1 for calling a global function with example usage: _bias_dropout_add_func(x_with_bias, residual, prob, training) and returns: out, out