docstr: This is implementation of paper
  "Pruning neural networks without any data by iteratively conserving synaptic flow"
  The score takes 5 steps:
      1. For each layer, for each parameter, calculate the absolute value |0|
      2. Use a single all-one-vector with dim = [1, c, h, w] to run a forward,
         Since only consider linear and Con2d operation, the forward output is multiple( [ |0l| for l in L] )
      3. New loss function R = sum(output), and then run backward
      4. for each layer, calculate Sl = Hadamard product( df/dw, w), where Sij=aij×bij
      5. score = sum( [ Sl for l in layers ] )
  Comments:
      1. this is data-Agnostic
      2. only compute on a single example
docstr_tok: ["This","is","implementation","of","paper","''","Pruning","neural","networks","without","any","data","by","iteratively","conserving","synaptic","flow","''","The","score","takes","5","steps",":","1",".","For","each","layer",",","for","each","parameter",",","calculate","the","absolute","value","|0|","2",".","Use","a","single","all-one-vector","with","dim","=","[","1",",","c",",","h",",","w","]","to","run","a","forward",",","Since","only","consider","linear","and","Con2d","operation",",","the","forward","output","is","multiple","(","[","|0l|","for","l","in","L","]",")","3",".","New","loss","function","R","=","sum","(","output",")",",","and","then","run","backward","4.","for","each","layer",",","calculate","Sl","=","Hadamard","product","(","df\/dw",",","w",")",",","where","Sij=aij\u00d7bij","5.","score","=","sum","(","[","Sl","for","l","in","layers","]",")","Comments",":","1.","this","is","data-Agnostic","2.","only","compute","on","a","single","example"]
code: def evaluate(
      self,
      arch,
      device,
      batch_data: object,
      batch_labels: torch.Tensor,
      space_name: str,
  ) -> float:
      """
      This is implementation of paper
      "Pruning neural networks without any data by iteratively conserving synaptic flow"
      The score takes 5 steps:
          1. For each layer, for each parameter, calculate the absolute value |0|
          2. Use a single all-one-vector with dim = [1, c, h, w] to run a forward,
             Since only consider linear and Con2d operation, the forward output is multiple( [ |0l| for l in L] )
          3. New loss function R = sum(output), and then run backward
          4. for each layer, calculate Sl = Hadamard product( df/dw, w), where Sij=aij×bij
          5. score = sum( [ Sl for l in layers ] )
      Comments:
          1. this is data-Agnostic
          2. only compute on a single example
      """
  
      ### singa configs
      mssgd = MSSGD(
          lr=0.005,
          momentum=0.9,
          weight_decay=1e-5,
          dtype=singa_dtype["float32"],
      )
      device_id = 0
      max_epoch = 1
      model = arch
      graph = True
      verbosity = 0
      dist_option = "plain"
      spars = None
      precision = "float32"
      global_rank = 0
      world_size = 1
  
      ### singa setups
      # print ("device: \n", device)
      if device == "cpu":
          dev = singa_device.get_default_device()
      else:  # GPU
          dev = singa_device.create_cuda_gpu_on(
              local_rank
          )  # need to change to CPU device for CPU-only machines
      dev.SetRandSeed(0)
      np.random.seed(0)
  
      # For distributed training, sequential has better performance
      if hasattr(mssgd, "communicator"):
          DIST = True
          sequential = True
      else:
          DIST = False
          sequential = False
  
      model.train()
  
      ### process batch_data
      x = batch_data.cpu().numpy()  # Size([1, 100]) and all ones
      x = x.astype(np_dtype[precision])
      y = np.ones(x.shape[0], dtype=np.int32)
      if model.dimension == 2:  # input data dimension
          tx = tensor.Tensor(x.shape, dev, singa_dtype[precision])
      ty = tensor.Tensor((x.shape[0],), dev, tensor.int32)
  
      model.set_optimizer(mssgd)
      model.compile(
          [tx], is_train=True, use_graph=graph, sequential=sequential
      )
      dev.SetVerbosity(verbosity)
  
      # 1. Convert params to their abs.
      synflow_flag = (
          True  ### just change the model to the absolute value
      )
      tx.copy_from_numpy(x)  # dtype=np.float32
      ty.copy_from_numpy(y)
      # print ("before model forward ...")
      pn_p_g_list, out, loss = model(
          tx, ty, dist_option, spars, synflow_flag
      )
      # print ("---------------------------------------")
      # print ("before absolute prune_synflow !!!nemb input vector!!! tensor.to_numpy(loss)[0]: ", tensor.to_numpy(loss)[0])
      # print ("before absolute prune_synflow !!!nemb input vector!!! tensor.to_numpy(loss): ", tensor.to_numpy(loss))
      # train_correct += accuracy(tensor.to_numpy(out), y)
      # train_loss += tensor.to_numpy(loss)[0]
      # all params turned to positive
      for pn_p_g_item in pn_p_g_list:
          # print ("absolute value parameter name: \n", pn_p_g_item[0])
          param_np = tensor.to_numpy(pn_p_g_item[1])
          # print ("param_np shape: \n", param_np.shape)
          # print ("param_np sqrt norm: \n", np.sqrt(LA.norm(param_np)/param_np.size))
          # print ("before abs np.min(tensor.to_numpy(pn_p_g_item[1])): \n", np.min(tensor.to_numpy(pn_p_g_item[1])))
          pn_p_g_item[1] = tensor.abs(
              pn_p_g_item[1]
          )  # tensor actually ..
          # print ("after abs np.min(tensor.to_numpy(pn_p_g_item[1])): \n", np.min(tensor.to_numpy(pn_p_g_item[1])))
          # print ("after abs pn_p_g_item[1][0]: \n", pn_p_g_item[1][0])
  
      # 2. Compute gradients with input of one dummy example ( 1-vector with dimension [1, c, h, w] )
      # 3.R = sum(output)
      # 4. Select the gradients that we want to use for search/prune
      # 5. Sum over all parameter's results to get the final score.
      # score = sum([grad.sum() for grad in grads_abs])
  
      # print ("calculate synflow")
      synflow_flag = True
      ### step 1: all one input
      # Copy the patch data into input tensors
      # tx.copy_from_numpy(np.ones(x.shape, dtype=np.float32))
      tx.copy_from_numpy(
          x
      )  # dtype=np.float32 # actually it is all ones ... --> np.ones(x.shape, dtype=np.float32)
      ty.copy_from_numpy(y)
      ### step 2: all weights turned to positive (done)
      ### step 3: new loss (done)
      # print ("before model forward ...")
      pn_p_g_list, out, loss = model(
          tx, ty, dist_option, spars, synflow_flag
      )
      # print ("prune_synflow !!!nemb input vector!!! synflow step tensor.to_numpy(loss)[0]: ", tensor.to_numpy(loss)[0])
      ### step 4: calculate the multiplication of weights
      score = 0.0
      for pn_p_g_item in pn_p_g_list:
          # print ("calculate weight param * grad parameter name: \n", pn_p_g_item[0])
          if (
              len(pn_p_g_item[1].shape) == 2
          ):  # param_value.data is "weight"
              # print ("pn_p_g_item[1].shape: \n", pn_p_g_item[1].shape)
              # print ("tensor.to_numpy(pn_p_g_item[1][0]): ", tensor.to_numpy(pn_p_g_item[1][0]))
              # print ("calculate synflow parameter name: \n", pn_p_g_item[0])
              # print ("should be positive np.min(tensor.to_numpy(pn_p_g_item[1])): ", np.min(tensor.to_numpy(pn_p_g_item[1])))
              # print ("weight should be positive tensor.to_numpy(pn_p_g_item[1][0])[0, :10]: ", tensor.to_numpy(pn_p_g_item[1][0])[0, :10])
              # print ("gradients tensor.to_numpy(pn_p_g_item[2][0])[0, :10]: ", tensor.to_numpy(pn_p_g_item[2][0])[0, :10])
              # print ()
              score += np.sum(
                  np.absolute(
                      tensor.to_numpy(pn_p_g_item[1])
                      * tensor.to_numpy(pn_p_g_item[2])
                  )
              )
      # print ("layer_hidden_list: \n", layer_hidden_list)
      # print ("prune_synflow !!!one-hot input vector!!! absolute step tensor.to_numpy(loss)[0]: ", tensor.to_numpy(loss)[0])
      print("score: \n", score)
  
      return score
  
imports: from src.eva_engine.phase1.algo.alg_base import Evaluator
  from src.common.constant import Config
  from singa import singa_wrap
  from singa import device
  from singa import tensor
  from singa import opt
  from singa import autograd
  from singa.opt import Optimizer
  from singa.opt import DecayScheduler
  from singa.opt import Constant
  import numpy
  import time
  import argparse
  from PIL import Image
  from numpy import linalg
  import torch
label_desc: Use image node_id 2 for calling the SynFlowEvaluator obj's underlying member method code with example usage: obj.evaluate(arch, device, batch_data, batch_labels, space_name) and returns: score