class_docstr: ## FFN module
class_docstr_tok: ["#","#","FFN","module"]
docstr: * `d_model` is the number of features in a token embedding
  * `d_ff` is the number of features in the hidden layer of the FFN
  * `dropout` is dropout probability for the hidden layer
  * `is_gated` specifies whether the hidden layer is gated
  * `bias1` specified whether the first fully connected layer should have a learnable bias
  * `bias2` specified whether the second fully connected layer should have a learnable bias
  * `bias_gate` specified whether the fully connected layer for the gate should have a learnable bias
docstr_tok: ["*","`","d_model","`","is","the","number","of","features","in","a","token","embedding","*","`","d_ff","`","is","the","number","of","features","in","the","hidden","layer","of","the","FFN","*","`","dropout","`","is","dropout","probability","for","the","hidden","layer","*","`","is_gated","`","specifies","whether","the","hidden","layer","is","gated","*","`","bias1","`","specified","whether","the","first","fully","connected","layer","should","have","a","learnable","bias","*","`","bias2","`","specified","whether","the","second","fully","connected","layer","should","have","a","learnable","bias","*","`","bias_gate","`","specified","whether","the","fully","connected","layer","for","the","gate","should","have","a","learnable","bias"]
code: def __init__(
      self,
      d_model: int,
      d_ff: int,
      dropout: float = 0.1,
      activation=nn.ReLU(),
      is_gated: bool = False,
      bias1: bool = True,
      bias2: bool = True,
      bias_gate: bool = True,
  ):
      """
      * `d_model` is the number of features in a token embedding
      * `d_ff` is the number of features in the hidden layer of the FFN
      * `dropout` is dropout probability for the hidden layer
      * `is_gated` specifies whether the hidden layer is gated
      * `bias1` specified whether the first fully connected layer should have a learnable bias
      * `bias2` specified whether the second fully connected layer should have a learnable bias
      * `bias_gate` specified whether the fully connected layer for the gate should have a learnable bias
      """
      super().__init__()
      # Layer one parameterized by weight $W_1$ and bias $b_1$
      self.layer1 = nn.Linear(d_model, d_ff, bias=bias1)
      # Layer one parameterized by weight $W_1$ and bias $b_1$
      self.layer2 = nn.Linear(d_ff, d_model, bias=bias2)
      # Hidden layer dropout
      self.dropout = nn.Dropout(dropout)
      # Activation function $f$
      self.activation = activation
      # Whether there is a gate
      self.is_gated = is_gated
      if is_gated:
          # If there is a gate the linear layer to transform inputs to
          # be multiplied by the gate, parameterized by weight $V$ and bias $c$
          self.linear_v = nn.Linear(d_model, d_ff, bias=bias_gate)
  
imports: import torch
  from torch import nn
  from labml_helpers.module import Module
label_desc: Use image node_id 1 to create a new FeedForward object from inherited base classes: Module with example: obj = FeedForward(d_model, d_ff, dropout, activation, is_gated, bias1, bias2, bias_gate)