class_docstr: ## Dueling Network ⚔️ Model for $Q$ Values
  
  We are using a [dueling network](https://arxiv.org/abs/1511.06581)
   to calculate Q-values.
  Intuition behind dueling network architecture is that in most states
   the action doesn't matter,
  and in some states the action is significant. Dueling network allows
   this to be represented very well.
  
  egin{align}
      Q^\pi(s,a) &= V^\pi(s) + A^\pi(s, a)
      \
      \mathop{\mathbb{E}}_{a \sim \pi(s)}
       \Big[
        A^\pi(s, a)
       \Big]
      &= 0
  \end{align}
  
  So we create two networks for $V$ and $A$ and get $Q$ from them.
  $$
      Q(s, a) = V(s) +
      \Big(
          A(s, a) - rac{1}{|\mathcal{A}|} \sum_{a' \in \mathcal{A}} A(s, a')
      \Big)
  $$
  We share the initial layers of the $V$ and $A$ networks.
class_docstr_tok: ["#","#","Dueling","Network","\u2694\ufe0f","Model","for","$","Q","$","Values","We","are","using","a","[","dueling","network","]","(","https",":","\/\/arxiv.org\/abs\/1511.06581",")","to","calculate","Q-values",".","Intuition","behind","dueling","network","architecture","is","that","in","most","states","the","action","does","n't","matter",",","and","in","some","states","the","action","is","significant",".","Dueling","network","allows","this","to","be","represented","very","well",".","\begin","{","align","}","Q^\\pi","(","s",",","a",")","&","=","V^\\pi","(","s",")","+","A^\\pi","(","s",",","a",")","\\","\\mathop","{","\\mathbb","{","E","}","}","_","{","a","\\sim","\\pi","(","s",")","}","\\Big","[","A^\\pi","(","s",",","a",")","\\Big","]","&","=","0","\\end","{","align","}","So","we","create","two","networks","for","$","V","$","and","$","A","$","and","get","$","Q","$","from","them",".","$","$","Q","(","s",",","a",")","=","V","(","s",")","+","\\Big","(","A","(","s",",","a",")","-","rac","{","1","}","{","|\\mathcal","{","A","}","|","}","\\sum_","{","a","'","\\in","\\mathcal","{","A","}","}","A","(","s",",","a","'",")","\\Big",")","$","$","We","share","the","initial","layers","of","the","$","V","$","and","$","A","$","networks","."]
code: def __init__(self):
      super().__init__()
      self.conv = nn.Sequential(
          # The first convolution layer takes a
          # $84\times84$ frame and produces a $20\times20$ frame
          nn.Conv2d(
              in_channels=4, out_channels=32, kernel_size=8, stride=4
          ),
          nn.ReLU(),
          # The second convolution layer takes a
          # $20\times20$ frame and produces a $9\times9$ frame
          nn.Conv2d(
              in_channels=32, out_channels=64, kernel_size=4, stride=2
          ),
          nn.ReLU(),
          # The third convolution layer takes a
          # $9\times9$ frame and produces a $7\times7$ frame
          nn.Conv2d(
              in_channels=64, out_channels=64, kernel_size=3, stride=1
          ),
          nn.ReLU(),
      )
  
      # A fully connected layer takes the flattened
      # frame from third convolution layer, and outputs
      # $512$ features
      self.lin = nn.Linear(in_features=7 * 7 * 64, out_features=512)
      self.activation = nn.ReLU()
  
      # This head gives the state value $V$
      self.state_value = nn.Sequential(
          nn.Linear(in_features=512, out_features=256),
          nn.ReLU(),
          nn.Linear(in_features=256, out_features=1),
      )
      # This head gives the action value $A$
      self.action_value = nn.Sequential(
          nn.Linear(in_features=512, out_features=256),
          nn.ReLU(),
          nn.Linear(in_features=256, out_features=4),
      )
  
imports: import torch
  from torch import nn
  from labml_helpers.module import Module
label_desc: Use image node_id 1 to create a new Model object from inherited base classes: Module with example: obj = Model()