class_docstr: Implementation of a model for inference composable with ragged batching.
class_docstr_tok: ["Implementation","of","a","model","for","inference","composable","with","ragged","batching","."]
docstr: After completing a forward pass, determine whether or not the there are any KV blocks
  that maybe freed since they are no longer in use.
  
  Consider the following example:
  
  We have a block size of 4 and a local window size of 8. At the beginning of the forward
  pass there 10 tokens had been seen and the new forward has a size of 4. This would lend
  itself to the following cache structure prior to the forward:
      [[0, 1, 2*, 3*] [4*, 5*, 6*, 7*] [8*, 9*, x, x] [x x x x]]
  Where x's denote empty cache locations and * denote values that are needed for attention
  of the next open slot. After the forward, the cache would look like the following:
      [[0, 1, 2, 3] [4, 5, 6*, 7*] [8*, 9*, 10*, 11*] [12* 13* x x]]
  In this case, the first block is no longer needed since it is not needed for any future
  local attention windows. This function would be responsible for freeing that block.
  
  Default behavior assumes no local patterns that require freeing and in general should
  be sufficient.
docstr_tok: ["After","completing","a","forward","pass",",","determine","whether","or","not","the","there","are","any","KV","blocks","that","maybe","freed","since","they","are","no","longer","in","use",".","Consider","the","following","example",":","We","have","a","block","size","of","4","and","a","local","window","size","of","8",".","At","the","beginning","of","the","forward","pass","there","10","tokens","had","been","seen","and","the","new","forward","has","a","size","of","4",".","This","would","lend","itself","to","the","following","cache","structure","prior","to","the","forward",":","[","[","0",",","1",",","2","*",",","3","*","]","[","4","*",",","5","*",",","6","*",",","7","*","]","[","8","*",",","9","*",",","x",",","x","]","[","x","x","x","x","]","]","Where","x","'s","denote","empty","cache","locations","and","*","denote","values","that","are","needed","for","attention","of","the","next","open","slot",".","After","the","forward",",","the","cache","would","look","like","the","following",":","[","[","0",",","1",",","2",",","3","]","[","4",",","5",",","6","*",",","7","*","]","[","8","*",",","9","*",",","10","*",",","11","*","]","[","12","*","13","*","x","x","]","]","In","this","case",",","the","first","block","is","no","longer","needed","since","it","is","not","needed","for","any","future","local","attention","windows",".","This","function","would","be","responsible","for","freeing","that","block",".","Default","behavior","assumes","no","local","patterns","that","require","freeing","and","in","general","should","be","sufficient","."]
code: def maybe_free_kv(self, sequence: DSSequenceDescriptor) -> None:
      """
      After completing a forward pass, determine whether or not the there are any KV blocks
      that maybe freed since they are no longer in use.
  
      Consider the following example:
  
      We have a block size of 4 and a local window size of 8. At the beginning of the forward
      pass there 10 tokens had been seen and the new forward has a size of 4. This would lend
      itself to the following cache structure prior to the forward:
          [[0, 1, 2*, 3*] [4*, 5*, 6*, 7*] [8*, 9*, x, x] [x x x x]]
      Where x's denote empty cache locations and * denote values that are needed for attention
      of the next open slot. After the forward, the cache would look like the following:
          [[0, 1, 2, 3] [4, 5, 6*, 7*] [8*, 9*, 10*, 11*] [12* 13* x x]]
      In this case, the first block is no longer needed since it is not needed for any future
      local attention windows. This function would be responsible for freeing that block.
  
      Default behavior assumes no local patterns that require freeing and in general should
      be sufficient.
      """
      pass
  
imports: from abc import ABC, abstractmethod
  from typing import Iterable, Optional, Tuple, Type
  import torch
  import deepspeed.comm
  from ..ragged import DSStateManager, RaggedBatchWrapper
  from ..ragged.manager_configs import KVCacheConfig
  from ..ragged import DSSequenceDescriptor
  from ..model_implementations.layer_container_base import LayerContainer
  from ..config_v2 import RaggedInferenceEngineConfig
  from .flat_model_helpers import ModelMetadata
label_desc: Use image node_id 15 for calling the DSInferenceModelBase obj's underlying member method code with example usage: obj.maybe_free_kv(sequence) without return types