class_docstr: Pipeline for text-guided video-to-video generation.
  
  This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
  implemented for all pipelines (downloading, saving, running on a particular device, etc.).
  
  Args:
      vae ([`AutoencoderKL`]):
          Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations.
      text_encoder ([`CLIPTextModel`]):
          Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
      tokenizer (`CLIPTokenizer`):
          A [`~transformers.CLIPTokenizer`] to tokenize text.
      unet ([`UNet3DConditionModel`]):
          A [`UNet3DConditionModel`] to denoise the encoded video latents.
      scheduler ([`SchedulerMixin`]):
          A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
          [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
class_docstr_tok: ["Pipeline","for","text-guided","video-to-video","generation",".","This","model","inherits","from","[","`","DiffusionPipeline","`","]",".","Check","the","superclass","documentation","for","the","generic","methods","implemented","for","all","pipelines","(","downloading",",","saving",",","running","on","a","particular","device",",","etc.",")",".","Args",":","vae","(","[","`","AutoencoderKL","`","]",")",":","Variational","Auto-Encoder","(","VAE",")","Model","to","encode","and","decode","videos","to","and","from","latent","representations",".","text_encoder","(","[","`","CLIPTextModel","`","]",")",":","Frozen","text-encoder","(","[","clip-vit-large-patch14","]","(","https",":","\/\/huggingface.co\/openai\/clip-vit-large-patch14",")",")",".","tokenizer","(","`","CLIPTokenizer","`",")",":","A","[","`","~transformers.CLIPTokenizer","`","]","to","tokenize","text",".","unet","(","[","`","UNet3DConditionModel","`","]",")",":","A","[","`","UNet3DConditionModel","`","]","to","denoise","the","encoded","video","latents",".","scheduler","(","[","`","SchedulerMixin","`","]",")",":","A","scheduler","to","be","used","in","combination","with","`","unet","`","to","denoise","the","encoded","image","latents",".","Can","be","one","of","[","`","DDIMScheduler","`","]",",","[","`","LMSDiscreteScheduler","`","]",",","or","[","`","PNDMScheduler","`","]","."]
docstr: The call function to the pipeline for generation.
  
  Args:
      prompt (`str` or `List[str]`, *optional*):
          The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
      video (`List[np.ndarray]` or `torch.FloatTensor`):
          `video` frames or tensor representing a video batch to be used as the starting point for the process.
          Can also accept video latents as `image`, if passing latents directly, it will not be encoded again.
      strength (`float`, *optional*, defaults to 0.8):
          Indicates extent to transform the reference `video`. Must be between 0 and 1. `video` is used as a
          starting point, adding more noise to it the larger the `strength`. The number of denoising steps
          depends on the amount of noise initially added. When `strength` is 1, added noise is maximum and the
          denoising process runs for the full number of iterations specified in `num_inference_steps`. A value of
          1 essentially ignores `video`.
      num_inference_steps (`int`, *optional*, defaults to 50):
          The number of denoising steps. More denoising steps usually lead to a higher quality videos at the
          expense of slower inference.
      guidance_scale (`float`, *optional*, defaults to 7.5):
          A higher guidance scale value encourages the model to generate images closely linked to the text
          `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
      negative_prompt (`str` or `List[str]`, *optional*):
          The prompt or prompts to guide what to not include in video generation. If not defined, you need to
          pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
      eta (`float`, *optional*, defaults to 0.0):
          Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies
          to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
      generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
          A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
          generation deterministic.
      latents (`torch.FloatTensor`, *optional*):
          Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video
          generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
          tensor is generated by sampling using the supplied random `generator`. Latents should be of shape
          `(batch_size, num_channel, num_frames, height, width)`.
      prompt_embeds (`torch.FloatTensor`, *optional*):
          Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
          provided, text embeddings are generated from the `prompt` input argument.
      negative_prompt_embeds (`torch.FloatTensor`, *optional*):
          Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
          not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
      output_type (`str`, *optional*, defaults to `"np"`):
          The output format of the generated video. Choose between `torch.FloatTensor` or `np.array`.
      return_dict (`bool`, *optional*, defaults to `True`):
          Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead
          of a plain tuple.
      callback (`Callable`, *optional*):
          A function that calls every `callback_steps` steps during inference. The function is called with the
          following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
      callback_steps (`int`, *optional*, defaults to 1):
          The frequency at which the `callback` function is called. If not specified, the callback is called at
          every step.
      cross_attention_kwargs (`dict`, *optional*):
          A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
          [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
      clip_skip (`int`, *optional*):
          Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
          the output of the pre-final layer will be used for computing the prompt embeddings.
  Examples:
  
  Returns:
      [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] or `tuple`:
          If `return_dict` is `True`, [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] is
          returned, otherwise a `tuple` is returned where the first element is a list with the generated frames.
docstr_tok: ["The","call","function","to","the","pipeline","for","generation",".","Args",":","prompt","(","`","str","`","or","`","List","[","str","]","`",",","*","optional","*",")",":","The","prompt","or","prompts","to","guide","image","generation",".","If","not","defined",",","you","need","to","pass","`","prompt_embeds","`",".","video","(","`","List","[","np.ndarray","]","`","or","`","torch.FloatTensor","`",")",":","`","video","`","frames","or","tensor","representing","a","video","batch","to","be","used","as","the","starting","point","for","the","process",".","Can","also","accept","video","latents","as","`","image","`",",","if","passing","latents","directly",",","it","will","not","be","encoded","again",".","strength","(","`","float","`",",","*","optional","*",",","defaults","to","0.8",")",":","Indicates","extent","to","transform","the","reference","`","video","`",".","Must","be","between","0","and","1",".","`","video","`","is","used","as","a","starting","point",",","adding","more","noise","to","it","the","larger","the","`","strength","`",".","The","number","of","denoising","steps","depends","on","the","amount","of","noise","initially","added",".","When","`","strength","`","is","1",",","added","noise","is","maximum","and","the","denoising","process","runs","for","the","full","number","of","iterations","specified","in","`","num_inference_steps","`",".","A","value","of","1","essentially","ignores","`","video","`",".","num_inference_steps","(","`","int","`",",","*","optional","*",",","defaults","to","50",")",":","The","number","of","denoising","steps",".","More","denoising","steps","usually","lead","to","a","higher","quality","videos","at","the","expense","of","slower","inference",".","guidance_scale","(","`","float","`",",","*","optional","*",",","defaults","to","7.5",")",":","A","higher","guidance","scale","value","encourages","the","model","to","generate","images","closely","linked","to","the","text","`","prompt","`","at","the","expense","of","lower","image","quality",".","Guidance","scale","is","enabled","when","`","guidance_scale",">","1","`",".","negative_prompt","(","`","str","`","or","`","List","[","str","]","`",",","*","optional","*",")",":","The","prompt","or","prompts","to","guide","what","to","not","include","in","video","generation",".","If","not","defined",",","you","need","to","pass","`","negative_prompt_embeds","`","instead",".","Ignored","when","not","using","guidance","(","`","guidance_scale","<","1","`",")",".","eta","(","`","float","`",",","*","optional","*",",","defaults","to","0.0",")",":","Corresponds","to","parameter","eta","(","\u03b7",")","from","the","[","DDIM","]","(","https",":","\/\/arxiv.org\/abs\/2010.02502",")","paper",".","Only","applies","to","the","[","`","~schedulers.DDIMScheduler","`","]",",","and","is","ignored","in","other","schedulers",".","generator","(","`","torch.Generator","`","or","`","List","[","torch.Generator","]","`",",","*","optional","*",")",":","A","[","`","torch.Generator","`","]","(","https",":","\/\/pytorch.org\/docs\/stable\/generated\/torch.Generator.html",")","to","make","generation","deterministic",".","latents","(","`","torch.FloatTensor","`",",","*","optional","*",")",":","Pre-generated","noisy","latents","sampled","from","a","Gaussian","distribution",",","to","be","used","as","inputs","for","video","generation",".","Can","be","used","to","tweak","the","same","generation","with","different","prompts",".","If","not","provided",",","a","latents","tensor","is","generated","by","sampling","using","the","supplied","random","`","generator","`",".","Latents","should","be","of","shape","`","(","batch_size",",","num_channel",",","num_frames",",","height",",","width",")","`",".","prompt_embeds","(","`","torch.FloatTensor","`",",","*","optional","*",")",":","Pre-generated","text","embeddings",".","Can","be","used","to","easily","tweak","text","inputs","(","prompt","weighting",")",".","If","not","provided",",","text","embeddings","are","generated","from","the","`","prompt","`","input","argument",".","negative_prompt_embeds","(","`","torch.FloatTensor","`",",","*","optional","*",")",":","Pre-generated","negative","text","embeddings",".","Can","be","used","to","easily","tweak","text","inputs","(","prompt","weighting",")",".","If","not","provided",",","`","negative_prompt_embeds","`","are","generated","from","the","`","negative_prompt","`","input","argument",".","output_type","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","np","''","`",")",":","The","output","format","of","the","generated","video",".","Choose","between","`","torch.FloatTensor","`","or","`","np.array","`",".","return_dict","(","`","bool","`",",","*","optional","*",",","defaults","to","`","True","`",")",":","Whether","or","not","to","return","a","[","`","~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput","`","]","instead","of","a","plain","tuple",".","callback","(","`","Callable","`",",","*","optional","*",")",":","A","function","that","calls","every","`","callback_steps","`","steps","during","inference",".","The","function","is","called","with","the","following","arguments",":","`","callback","(","step",":","int",",","timestep",":","int",",","latents",":","torch.FloatTensor",")","`",".","callback_steps","(","`","int","`",",","*","optional","*",",","defaults","to","1",")",":","The","frequency","at","which","the","`","callback","`","function","is","called",".","If","not","specified",",","the","callback","is","called","at","every","step",".","cross_attention_kwargs","(","`","dict","`",",","*","optional","*",")",":","A","kwargs","dictionary","that","if","specified","is","passed","along","to","the","[","`","AttentionProcessor","`","]","as","defined","in","[","`","self.processor","`","]","(","https",":","\/\/github.com\/huggingface\/diffusers\/blob\/main\/src\/diffusers\/models\/attention_processor.py",")",".","clip_skip","(","`","int","`",",","*","optional","*",")",":","Number","of","layers","to","be","skipped","from","CLIP","while","computing","the","prompt","embeddings",".","A","value","of","1","means","that","the","output","of","the","pre-final","layer","will","be","used","for","computing","the","prompt","embeddings",".","Examples",":","Returns",":","[","`","~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput","`","]","or","`","tuple","`",":","If","`","return_dict","`","is","`","True","`",",","[","`","~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput","`","]","is","returned",",","otherwise","a","`","tuple","`","is","returned","where","the","first","element","is","a","list","with","the","generated","frames","."]
code: def __call__(
      self,
      prompt: Union[str, List[str]] = None,
      video: Union[List[np.ndarray], torch.FloatTensor] = None,
      strength: float = 0.6,
      num_inference_steps: int = 50,
      guidance_scale: float = 15.0,
      negative_prompt: Optional[Union[str, List[str]]] = None,
      eta: float = 0.0,
      generator: Optional[
          Union[torch.Generator, List[torch.Generator]]
      ] = None,
      latents: Optional[torch.FloatTensor] = None,
      prompt_embeds: Optional[torch.FloatTensor] = None,
      negative_prompt_embeds: Optional[torch.FloatTensor] = None,
      output_type: Optional[str] = "np",
      return_dict: bool = True,
      callback: Optional[
          Callable[[int, int, torch.FloatTensor], None]
      ] = None,
      callback_steps: int = 1,
      cross_attention_kwargs: Optional[Dict[str, Any]] = None,
      clip_skip: Optional[int] = None,
  ):
      r"""
      The call function to the pipeline for generation.
  
      Args:
          prompt (`str` or `List[str]`, *optional*):
              The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
          video (`List[np.ndarray]` or `torch.FloatTensor`):
              `video` frames or tensor representing a video batch to be used as the starting point for the process.
              Can also accept video latents as `image`, if passing latents directly, it will not be encoded again.
          strength (`float`, *optional*, defaults to 0.8):
              Indicates extent to transform the reference `video`. Must be between 0 and 1. `video` is used as a
              starting point, adding more noise to it the larger the `strength`. The number of denoising steps
              depends on the amount of noise initially added. When `strength` is 1, added noise is maximum and the
              denoising process runs for the full number of iterations specified in `num_inference_steps`. A value of
              1 essentially ignores `video`.
          num_inference_steps (`int`, *optional*, defaults to 50):
              The number of denoising steps. More denoising steps usually lead to a higher quality videos at the
              expense of slower inference.
          guidance_scale (`float`, *optional*, defaults to 7.5):
              A higher guidance scale value encourages the model to generate images closely linked to the text
              `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
          negative_prompt (`str` or `List[str]`, *optional*):
              The prompt or prompts to guide what to not include in video generation. If not defined, you need to
              pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
          eta (`float`, *optional*, defaults to 0.0):
              Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies
              to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
          generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
              A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
              generation deterministic.
          latents (`torch.FloatTensor`, *optional*):
              Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video
              generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
              tensor is generated by sampling using the supplied random `generator`. Latents should be of shape
              `(batch_size, num_channel, num_frames, height, width)`.
          prompt_embeds (`torch.FloatTensor`, *optional*):
              Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
              provided, text embeddings are generated from the `prompt` input argument.
          negative_prompt_embeds (`torch.FloatTensor`, *optional*):
              Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
              not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
          output_type (`str`, *optional*, defaults to `"np"`):
              The output format of the generated video. Choose between `torch.FloatTensor` or `np.array`.
          return_dict (`bool`, *optional*, defaults to `True`):
              Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead
              of a plain tuple.
          callback (`Callable`, *optional*):
              A function that calls every `callback_steps` steps during inference. The function is called with the
              following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
          callback_steps (`int`, *optional*, defaults to 1):
              The frequency at which the `callback` function is called. If not specified, the callback is called at
              every step.
          cross_attention_kwargs (`dict`, *optional*):
              A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
              [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
          clip_skip (`int`, *optional*):
              Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
              the output of the pre-final layer will be used for computing the prompt embeddings.
      Examples:
  
      Returns:
          [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] or `tuple`:
              If `return_dict` is `True`, [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] is
              returned, otherwise a `tuple` is returned where the first element is a list with the generated frames.
      """
      # 0. Default height and width to unet
      num_images_per_prompt = 1
  
      # 1. Check inputs. Raise error if not correct
      self.check_inputs(
          prompt,
          strength,
          callback_steps,
          negative_prompt,
          prompt_embeds,
          negative_prompt_embeds,
      )
  
      # 2. Define call parameters
      if prompt is not None and isinstance(prompt, str):
          batch_size = 1
      elif prompt is not None and isinstance(prompt, list):
          batch_size = len(prompt)
      else:
          batch_size = prompt_embeds.shape[0]
  
      device = self._execution_device
      # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
      # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
      # corresponds to doing no classifier free guidance.
      do_classifier_free_guidance = guidance_scale > 1.0
  
      # 3. Encode input prompt
      text_encoder_lora_scale = (
          cross_attention_kwargs.get("scale", None)
          if cross_attention_kwargs is not None
          else None
      )
      prompt_embeds, negative_prompt_embeds = self.encode_prompt(
          prompt,
          device,
          num_images_per_prompt,
          do_classifier_free_guidance,
          negative_prompt,
          prompt_embeds=prompt_embeds,
          negative_prompt_embeds=negative_prompt_embeds,
          lora_scale=text_encoder_lora_scale,
          clip_skip=clip_skip,
      )
      # For classifier free guidance, we need to do two forward passes.
      # Here we concatenate the unconditional and text embeddings into a single batch
      # to avoid doing two forward passes
      if do_classifier_free_guidance:
          prompt_embeds = torch.cat(
              [negative_prompt_embeds, prompt_embeds]
          )
  
      # 4. Preprocess video
      video = preprocess_video(video)
  
      # 5. Prepare timesteps
      self.scheduler.set_timesteps(num_inference_steps, device=device)
      timesteps, num_inference_steps = self.get_timesteps(
          num_inference_steps, strength, device
      )
      latent_timestep = timesteps[:1].repeat(
          batch_size * num_images_per_prompt
      )
  
      # 5. Prepare latent variables
      latents = self.prepare_latents(
          video,
          latent_timestep,
          batch_size,
          prompt_embeds.dtype,
          device,
          generator,
      )
  
      # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
      extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
  
      # 7. Denoising loop
      num_warmup_steps = (
          len(timesteps) - num_inference_steps * self.scheduler.order
      )
      with self.progress_bar(total=num_inference_steps) as progress_bar:
          for i, t in enumerate(timesteps):
              # expand the latents if we are doing classifier free guidance
              latent_model_input = (
                  torch.cat([latents] * 2)
                  if do_classifier_free_guidance
                  else latents
              )
              latent_model_input = self.scheduler.scale_model_input(
                  latent_model_input, t
              )
  
              # predict the noise residual
              noise_pred = self.unet(
                  latent_model_input,
                  t,
                  encoder_hidden_states=prompt_embeds,
                  cross_attention_kwargs=cross_attention_kwargs,
                  return_dict=False,
              )[0]
  
              # perform guidance
              if do_classifier_free_guidance:
                  noise_pred_uncond, noise_pred_text = noise_pred.chunk(
                      2
                  )
                  noise_pred = noise_pred_uncond + guidance_scale * (
                      noise_pred_text - noise_pred_uncond
                  )
  
              # reshape latents
              bsz, channel, frames, width, height = latents.shape
              latents = latents.permute(0, 2, 1, 3, 4).reshape(
                  bsz * frames, channel, width, height
              )
              noise_pred = noise_pred.permute(0, 2, 1, 3, 4).reshape(
                  bsz * frames, channel, width, height
              )
  
              # compute the previous noisy sample x_t -> x_t-1
              latents = self.scheduler.step(
                  noise_pred, t, latents, **extra_step_kwargs
              ).prev_sample
  
              # reshape latents back
              latents = (
                  latents[None, :]
                  .reshape(bsz, frames, channel, width, height)
                  .permute(0, 2, 1, 3, 4)
              )
  
              # call the callback, if provided
              if i == len(timesteps) - 1 or (
                  (i + 1) > num_warmup_steps
                  and (i + 1) % self.scheduler.order == 0
              ):
                  progress_bar.update()
                  if callback is not None and i % callback_steps == 0:
                      step_idx = i // getattr(
                          self.scheduler, "order", 1
                      )
                      callback(step_idx, t, latents)
  
      if output_type == "latent":
          return TextToVideoSDPipelineOutput(frames=latents)
  
      if (
          hasattr(self, "final_offload_hook")
          and self.final_offload_hook is not None
      ):
          self.unet.to("cpu")
  
      video_tensor = self.decode_latents(latents)
  
      if output_type == "pt":
          video = video_tensor
      else:
          video = tensor2vid(video_tensor)
  
      # Offload all models
      self.maybe_free_model_hooks()
  
      if not return_dict:
          return (video,)
  
      return TextToVideoSDPipelineOutput(frames=video)
  
imports: import inspect
  from typing import Any, Callable, Dict, List, Optional, Union
  import numpy
  import PIL.Image
  import torch
  from transformers import CLIPTextModel, CLIPTokenizer
  from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin
  from ...models import AutoencoderKL, UNet3DConditionModel
  from ...models.lora import adjust_lora_scale_text_encoder
  from ...schedulers import KarrasDiffusionSchedulers
  from ...utils import USE_PEFT_BACKEND, deprecate, logging, replace_example_docstring, scale_lora_layers, unscale_lora_layers
  from ...utils.torch_utils import randn_tensor
  from ..pipeline_utils import DiffusionPipeline
  from .None import TextToVideoSDPipelineOutput
label_desc: Use image node_id 15 for calling the VideoToVideoSDPipeline obj's underlying member method code with example usage: obj.__call__(prompt, video, strength, num_inference_steps, guidance_scale, negative_prompt, eta, generator, latents, prompt_embeds, negative_prompt_embeds, output_type, return_dict, callback, callback_steps, cross_attention_kwargs, clip_skip) and returns: TextToVideoSDPipelineOutput, TextToVideoSDPipelineOutput, video