class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
class_docstr_tok: ["Multi-headed","attention",".","See","``","Attention","Is","All","You","Need","''","for","more","details","."]
code: def _adaptive_prune_heads(
      self, reserve_head_index: List[Tuple[int, int]]
  ):
      new_q_weight = []
      new_q_bias = []
      new_k_weight = []
      new_k_bias = []
      new_v_weight = []
      new_v_bias = []
      new_out_proj_weight = []
  
      for ele in reserve_head_index:
          start_idx, end_idx = ele
          new_q_weight.append(self.q_proj.weight[start_idx:end_idx,])
          new_q_bias.append(self.q_proj.bias[start_idx:end_idx])
  
          new_k_weight.append(self.k_proj.weight[start_idx:end_idx,])
  
          new_k_bias.append(self.k_proj.bias[start_idx:end_idx])
  
          new_v_weight.append(self.v_proj.weight[start_idx:end_idx,])
          new_v_bias.append(self.v_proj.bias[start_idx:end_idx])
  
          new_out_proj_weight.append(
              self.out_proj.weight[:, start_idx:end_idx]
          )
  
      new_q_weight = torch.cat(new_q_weight).detach()
      new_k_weight = torch.cat(new_k_weight).detach()
      new_v_weight = torch.cat(new_v_weight).detach()
      new_out_proj_weight = torch.cat(
          new_out_proj_weight, dim=-1
      ).detach()
      new_q_weight.requires_grad = True
      new_k_weight.requires_grad = True
      new_v_weight.requires_grad = True
      new_out_proj_weight.requires_grad = True
  
      new_q_bias = torch.cat(new_q_bias).detach()
      new_q_bias.requires_grad = True
  
      new_k_bias = torch.cat(new_k_bias).detach()
      new_k_bias.requires_grad = True
  
      new_v_bias = torch.cat(new_v_bias).detach()
      new_v_bias.requires_grad = True
  
      self.q_proj.weight = torch.nn.Parameter(new_q_weight)
      self.q_proj.bias = torch.nn.Parameter(new_q_bias)
  
      self.k_proj.weight = torch.nn.Parameter(new_k_weight)
      self.k_proj.bias = torch.nn.Parameter(new_k_bias)
  
      self.v_proj.weight = torch.nn.Parameter(new_v_weight)
      self.v_proj.bias = torch.nn.Parameter(new_v_bias)
  
      self.out_proj.weight = torch.nn.Parameter(new_out_proj_weight)
  
      self.num_heads = len(reserve_head_index)
      self.embed_dim = self.head_dim * self.num_heads
      self.q_proj.out_features = self.embed_dim
      self.k_proj.out_features = self.embed_dim
      self.v_proj.out_features = self.embed_dim
  
imports: import math
  from typing import Dict, List, Optional, Tuple
  import torch
  import torch.nn.functional
  from torch import Tensor, nn
  from torch.nn import Parameter
  from fairseq import utils
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from fairseq.modules.quant_noise import quant_noise
  from fairseq.models.fairseq_incremental_decoder import FairseqIncrementalDecoder
label_desc: Use image node_id 5 for calling the MultiheadAttention obj's underlying member method code with example usage: obj._adaptive_prune_heads(reserve_head_index) without return types