docstr: Fine-tune OpenAI GPT models. Steps are roughly:
    - Analyze input data and modify it according to suggestions made by the OpenAI utility tool
    - Get a training and validation file
    - Determine base model to use
    - Submit a fine-tuning job via the OpenAI API
    - Monitor progress with exponential backoff (which has been modified for greater control given a time budget in hours),
    - Gather stats once fine-tuning finishes
    - Modify model metadata so that the new version triggers the fine-tuned version of the model (stored in the user's OpenAI account)
  
  Caveats:
    - As base fine-tuning models, OpenAI only supports the original GPT ones: `ada`, `babbage`, `curie`, `davinci`. This means if you fine-tune successively more than once, any fine-tuning other than the most recent one is lost.
    - A bunch of helper methods exist to be overridden in other handlers that follow the OpenAI API, e.g. Anyscale
docstr_tok: ["Fine-tune","OpenAI","GPT","models",".","Steps","are","roughly",":","-","Analyze","input","data","and","modify","it","according","to","suggestions","made","by","the","OpenAI","utility","tool","-","Get","a","training","and","validation","file","-","Determine","base","model","to","use","-","Submit","a","fine-tuning","job","via","the","OpenAI","API","-","Monitor","progress","with","exponential","backoff","(","which","has","been","modified","for","greater","control","given","a","time","budget","in","hours",")",",","-","Gather","stats","once","fine-tuning","finishes","-","Modify","model","metadata","so","that","the","new","version","triggers","the","fine-tuned","version","of","the","model","(","stored","in","the","user","'s","OpenAI","account",")","Caveats",":","-","As","base","fine-tuning","models",",","OpenAI","only","supports","the","original","GPT","ones",":","`","ada","`",",","`","babbage","`",",","`","curie","`",",","`","davinci","`",".","This","means","if","you","fine-tune","successively","more","than","once",",","any","fine-tuning","other","than","the","most","recent","one","is","lost",".","-","A","bunch","of","helper","methods","exist","to","be","overridden","in","other","handlers","that","follow","the","OpenAI","API",",","e.g",".","Anyscale"]
code: def finetune(
      self,
      df: Optional[pd.DataFrame] = None,
      args: Optional[Dict] = None,
  ) -> None:
      """
      Fine-tune OpenAI GPT models. Steps are roughly:
        - Analyze input data and modify it according to suggestions made by the OpenAI utility tool
        - Get a training and validation file
        - Determine base model to use
        - Submit a fine-tuning job via the OpenAI API
        - Monitor progress with exponential backoff (which has been modified for greater control given a time budget in hours),
        - Gather stats once fine-tuning finishes
        - Modify model metadata so that the new version triggers the fine-tuned version of the model (stored in the user's OpenAI account)
  
      Caveats:
        - As base fine-tuning models, OpenAI only supports the original GPT ones: `ada`, `babbage`, `curie`, `davinci`. This means if you fine-tune successively more than once, any fine-tuning other than the most recent one is lost.
        - A bunch of helper methods exist to be overridden in other handlers that follow the OpenAI API, e.g. Anyscale
      """  # noqa
  
      args = args if args else {}
      using_args = args.pop("using") if "using" in args else {}
      prompt_col = using_args.get("prompt_column", "prompt")
      completion_col = using_args.get("completion_column", "completion")
  
      api_key = get_api_key("openai", args, self.engine_storage)
      api_base = using_args.get(
          "api_base", os.environ.get("OPENAI_API_BASE", OPENAI_API_BASE)
      )
      org = using_args.get("api_organization")
      client = self._get_client(
          api_key=api_key, base_url=api_base, org=org
      )
  
      self._check_ft_cols(df, [prompt_col, completion_col])
  
      args = {**using_args, **args}
      prev_model_name = self.base_model_storage.json_get("args").get(
          "model_name", ""
      )
  
      if prev_model_name not in self.supported_ft_models:
          raise Exception(
              f"This model cannot be finetuned. Supported base models are {self.supported_ft_models}"
          )
  
      finetune_time = datetime.datetime.now().strftime(
          "%Y-%m-%d_%H-%M-%S"
      )
  
      temp_storage_path = tempfile.mkdtemp()
      temp_file_name = f"ft_{finetune_time}"
      temp_model_storage_path = (
          f"{temp_storage_path}/{temp_file_name}.jsonl"
      )
  
      file_names = self._prepare_ft_jsonl(
          df, temp_storage_path, temp_file_name, temp_model_storage_path
      )
  
      jsons = {k: None for k in file_names.keys()}
      for split, file_name in file_names.items():
          if os.path.isfile(os.path.join(temp_storage_path, file_name)):
              jsons[split] = client.files.create(
                  file=open(f"{temp_storage_path}/{file_name}", "rb"),
                  purpose="fine-tune",
              )
  
      if type(jsons["train"]) is openai.types.FileObject:
          train_file_id = jsons["train"].id
      else:
          train_file_id = jsons["base"].id
  
      if type(jsons["val"]) is openai.types.FileObject:
          val_file_id = jsons["val"].id
      else:
          val_file_id = None
  
      # `None` values are internally imputed by OpenAI to `null` or default values
      ft_params = {
          "training_file": train_file_id,
          "validation_file": val_file_id,
          "model": self._get_ft_model_type(prev_model_name),
      }
      ft_params = self._add_extra_ft_params(ft_params, using_args)
  
      start_time = datetime.datetime.now()
  
      ft_stats, result_file_id = self._ft_call(
          ft_params, client, args.get("hour_budget", 8)
      )
      ft_model_name = ft_stats.fine_tuned_model
  
      end_time = datetime.datetime.now()
      runtime = end_time - start_time
      name_extension = client.files.retrieve(
          file_id=result_file_id
      ).filename
      result_path = f"{temp_storage_path}/ft_{finetune_time}_result_{name_extension}"
  
      try:
          client.files.content(file_id=result_file_id).stream_to_file(
              result_path
          )
          if ".csv" in name_extension:
              # legacy endpoint
              train_stats = pd.read_csv(result_path)
              if "validation_token_accuracy" in train_stats.columns:
                  train_stats = train_stats[
                      train_stats["validation_token_accuracy"].notnull()
                  ]
              args["ft_api_info"] = ft_stats.dict()
              args["ft_result_stats"] = train_stats.to_dict()
  
          elif ".json" in name_extension:
              train_stats = pd.read_json(
                  path_or_buf=result_path, lines=True
              )  # new endpoint
              args["ft_api_info"] = args[
                  "ft_result_stats"
              ] = train_stats.to_dict()
  
      except Exception:
          logger.info(
              f"Error retrieving fine-tuning results. Please check manually for information on job {ft_stats.id} (result file {result_file_id})."
          )
  
      args["model_name"] = ft_model_name
      args["runtime"] = runtime.total_seconds()
      args["mode"] = self.base_model_storage.json_get("args").get(
          "mode", self.default_mode
      )
  
      self.model_storage.json_set("args", args)
      shutil.rmtree(temp_storage_path)
  
imports: import os
  import math
  import json
  import shutil
  import tempfile
  import datetime
  import textwrap
  import subprocess
  import concurrent.futures
  from typing import Optional, Dict
  import openai
  from openai import OpenAI
  import numpy
  import pandas
  from mindsdb.utilities.hooks import before_openai_query, after_openai_query
  from mindsdb.utilities import log
  from mindsdb.integrations.libs.base import BaseMLEngine
  from mindsdb.integrations.handlers.openai_handler.helpers import retry_with_exponential_backoff, truncate_msgs_for_token_limit, get_available_models, PendingFT
  from mindsdb.integrations.handlers.openai_handler.constants import CHAT_MODELS, IMAGE_MODELS, FINETUNING_MODELS, OPENAI_API_BASE
  from mindsdb.integrations.utilities.handler_utils import get_api_key
  from mindsdb.integrations.libs.llm_utils import get_completed_prompts
label_desc: Use image node_id 7 for calling the OpenAIHandler obj's underlying member method code with example usage: obj.finetune(df, args) without return types