code: def index_shard_and_quantize(self):
      assert os.path.exists(self.index_template_file)
      index = faiss.read_index(self.index_template_file)
      index_ivf = faiss.downcast_index(faiss.extract_index_ivf(index))
      assert self.nprobe <= index_ivf.quantizer.ntotal, (
          f"the number of vectors {index_ivf.quantizer.ntotal} is not enough"
          f" to retrieve {self.nprobe} neighbours, check."
      )
  
      if is_pretransform_index(index):
          d = index.chain.at(0).d_out
      else:
          d = self.input_d
      for i in range(0, self.nshards):
          sfn = f"{self.index_shard_prefix}{i}"
          cqfn = f"{self.coarse_quantization_prefix}{i}"  # fixme
          if os.path.exists(sfn) or os.path.exists(cqfn):
              logging.info(f"skipping shard: {i}")
              continue
          try:
              with open(cqfn, "xb") as cqf:
                  index.reset()
                  start = i * self.shard_size
                  j = 0
                  quantizer = faiss.index_cpu_to_all_gpus(
                      index_ivf.quantizer
                  )
                  for xb_j in tqdm(
                      self._iterate_transformed(
                          self.xb_ds,
                          start,
                          EMBEDDINGS_BATCH_SIZE,
                          np.float32,
                      ),
                      file=sys.stdout,
                  ):
                      assert xb_j.shape[1] == d
                      _, I = quantizer.search(xb_j, self.nprobe)
                      assert np.amin(I) >= 0, f"{I}"
                      assert np.amax(I) < index_ivf.nlist
                      cqf.write(I)
                      self._index_add_core_wrapper(  # fixme
                          index_ivf,
                          xb_j,
                          np.arange(
                              start + j, start + j + xb_j.shape[0]
                          ),
                          I[:, 0],
                      )
                      j += xb_j.shape[0]
                      assert j <= self.shard_size
                      if j == self.shard_size:
                          break
              logging.info(f"writing {sfn}...")
              faiss.write_index(index, sfn)
          except FileExistsError:
              logging.info(f"skipping shard: {i}")
              continue
      logging.info("done")
  
imports: import faiss
  import numpy
  import os
  from tqdm import tqdm, trange
  import sys
  import logging
  from faiss.contrib.ondisk import merge_ondisk
  from faiss.contrib.big_batch_search import big_batch_search
  from faiss.contrib.exhaustive_search import knn_ground_truth
  from faiss.contrib.evaluation import knn_intersection_measure
  from utils import get_intersection_cardinality_frequencies, margin, is_pretransform_index
  from dataset import create_dataset_from_oivf_config
label_desc: Use image node_id 6 for calling the OfflineIVF obj's underlying member method code with example usage: obj.index_shard_and_quantize() without return types