class_docstr: ## Dueling Network ⚔️ Model for $Q$ Values
  
  We are using a [dueling network](https://arxiv.org/abs/1511.06581)
   to calculate Q-values.
  Intuition behind dueling network architecture is that in most states
   the action doesn't matter,
  and in some states the action is significant. Dueling network allows
   this to be represented very well.
  
  egin{align}
      Q^\pi(s,a) &= V^\pi(s) + A^\pi(s, a)
      \
      \mathop{\mathbb{E}}_{a \sim \pi(s)}
       \Big[
        A^\pi(s, a)
       \Big]
      &= 0
  \end{align}
  
  So we create two networks for $V$ and $A$ and get $Q$ from them.
  $$
      Q(s, a) = V(s) +
      \Big(
          A(s, a) - rac{1}{|\mathcal{A}|} \sum_{a' \in \mathcal{A}} A(s, a')
      \Big)
  $$
  We share the initial layers of the $V$ and $A$ networks.
class_docstr_tok: ["#","#","Dueling","Network","\u2694\ufe0f","Model","for","$","Q","$","Values","We","are","using","a","[","dueling","network","]","(","https",":","\/\/arxiv.org\/abs\/1511.06581",")","to","calculate","Q-values",".","Intuition","behind","dueling","network","architecture","is","that","in","most","states","the","action","does","n't","matter",",","and","in","some","states","the","action","is","significant",".","Dueling","network","allows","this","to","be","represented","very","well",".","\begin","{","align","}","Q^\\pi","(","s",",","a",")","&","=","V^\\pi","(","s",")","+","A^\\pi","(","s",",","a",")","\\","\\mathop","{","\\mathbb","{","E","}","}","_","{","a","\\sim","\\pi","(","s",")","}","\\Big","[","A^\\pi","(","s",",","a",")","\\Big","]","&","=","0","\\end","{","align","}","So","we","create","two","networks","for","$","V","$","and","$","A","$","and","get","$","Q","$","from","them",".","$","$","Q","(","s",",","a",")","=","V","(","s",")","+","\\Big","(","A","(","s",",","a",")","-","rac","{","1","}","{","|\\mathcal","{","A","}","|","}","\\sum_","{","a","'","\\in","\\mathcal","{","A","}","}","A","(","s",",","a","'",")","\\Big",")","$","$","We","share","the","initial","layers","of","the","$","V","$","and","$","A","$","networks","."]
code: def forward(self, obs: torch.Tensor):
      # Convolution
      h = self.conv(obs)
      # Reshape for linear layers
      h = h.reshape((-1, 7 * 7 * 64))
  
      # Linear layer
      h = self.activation(self.lin(h))
  
      # $A$
      action_value = self.action_value(h)
      # $V$
      state_value = self.state_value(h)
  
      # $A(s, a) - \frac{1}{|\mathcal{A}|} \sum_{a' \in \mathcal{A}} A(s, a')$
      action_score_centered = action_value - action_value.mean(
          dim=-1, keepdim=True
      )
      # $Q(s, a) =V(s) + \Big(A(s, a) - \frac{1}{|\mathcal{A}|} \sum_{a' \in \mathcal{A}} A(s, a')\Big)$
      q = state_value + action_score_centered
  
      return q
  
imports: import torch
  from torch import nn
  from labml_helpers.module import Module
label_desc: Use image node_id 2 for calling the Model obj's underlying member method code with example usage: obj.forward(obs) and returns: q