class_docstr: This creates the function F such that the entire block can be
  expressed as F(G(X)). Includes pre-LayerNorm.
class_docstr_tok: ["This","creates","the","function","F","such","that","the","entire","block","can","be","expressed","as","F","(","G","(","X",")",")",".","Includes","pre-LayerNorm","."]
code: def __init__(
      self,
      dim,
      input_size,
      num_heads,
      cfg,
      dim_out=None,
      kernel_q=(1, 1, 1),
      kernel_kv=(1, 1, 1),
      stride_q=(1, 1, 1),
      stride_kv=(1, 1, 1),
      norm_layer=nn.LayerNorm,
  ):
      super().__init__()
      self.norm = norm_layer(dim, eps=1e-6, elementwise_affine=True)
  
      # This will be set externally during init
      self.thw = None
  
      # the actual attention details are the same as Multiscale
      # attention for MViTv2 (with channel up=projection inside block)
      # can also implement no upprojection attention for vanilla ViT
      self.attn = MultiScaleAttention(
          dim,
          dim_out,
          input_size=input_size,
          num_heads=num_heads,
          kernel_q=kernel_q,
          kernel_kv=kernel_kv,
          stride_q=stride_q,
          stride_kv=stride_kv,
          norm_layer=norm_layer,
          drop_rate=cfg.MVIT.DROPOUT_RATE,
          qkv_bias=cfg.MVIT.QKV_BIAS,
          has_cls_embed=cfg.MVIT.CLS_EMBED_ON,
          mode=cfg.MVIT.MODE,
          pool_first=cfg.MVIT.POOL_FIRST,
          rel_pos_spatial=cfg.MVIT.REL_POS_SPATIAL,
          rel_pos_temporal=cfg.MVIT.REL_POS_TEMPORAL,
          rel_pos_zero_init=cfg.MVIT.REL_POS_ZERO_INIT,
          residual_pooling=cfg.MVIT.RESIDUAL_POOLING,
          separate_qkv=cfg.MVIT.SEPARATE_QKV,
      )
  
imports: import sys
  from functools import partial
  import torch
  from torch import nn
  from torch.autograd import Function
  from slowfast.models.attention import MultiScaleAttention, attention_pool
  from slowfast.models.common import Mlp, TwoStreamFusion, drop_path
  from slowfast.models.utils import round_width
label_desc: Use image node_id 1 to create a new AttentionSubBlock object from inherited base classes: nn with example: obj = AttentionSubBlock(dim, input_size, num_heads, cfg, dim_out, kernel_q, kernel_kv, stride_q, stride_kv, norm_layer)