code: def get_params_for_weight_decay_optimization(
      model: Union[torch.nn.Module, List[torch.nn.Module]],
  ) -> Dict[str, torch.nn.Parameter]:
      """Divide params into with-weight-decay and without-weight-decay groups.
  
      Layernorms and biases will have no weight decay but the rest will.
      """
      modules = listify_model(model)
      weight_decay_params = {"params": []}
      no_weight_decay_params = {"params": [], "weight_decay": 0.0}
      for module in modules:
          for module_ in module.modules():
              if isinstance(
                  module_,
                  (FusedLayerNorm, FastLayerNorm, MixedFusedRMSNorm),
              ):
                  no_weight_decay_params["params"].extend(
                      [
                          p
                          for p in list(module_._parameters.values())
                          if p is not None
                      ]
                  )
              else:
                  weight_decay_params["params"].extend(
                      [
                          p
                          for n, p in list(module_._parameters.items())
                          if p is not None and n != "bias"
                      ]
                  )
                  no_weight_decay_params["params"].extend(
                      [
                          p
                          for n, p in list(module_._parameters.items())
                          if p is not None and n == "bias"
                      ]
                  )
  
      return weight_decay_params, no_weight_decay_params
  
imports: import itertools
  import math
  from typing import Dict, Iterator, List, Tuple, Union
  import torch
  import torch.nn
  from torch import Tensor
label_desc: Use image node_id 23 for calling a global function with example usage: get_params_for_weight_decay_optimization(model) and returns: weight_decay_params,  no_weight_decay_params