docstr: Creates a Dataset for pairs of sentences/paragraphs.
  
  The file needs to be in a column format,
  where each line has a column for the first sentence/paragraph, the second sentence/paragraph and the label
  seperated by e.g. '\t' (just like in the glue RTE-dataset https://gluebenchmark.com/tasks) .
  For each data pair we create a flair.data.DataPair object.
  
  Args:
      path_to_data: path to the data file
      columns: list of integers that indicate the respective columns. The first entry is the column for the first sentence, the second for the second sentence and the third for the label. Default [0,1,2]
      max_tokens_per_doc: If set, shortens sentences to this maximum number of tokens
      max_chars_per_doc: If set, shortens sentences to this maximum number of characters
      use_tokenizer: Whether to use in-built tokenizer
      in_memory: If True, data will be saved in list of flair.data.DataPair objects, otherwise we use lists with simple strings which needs less space
      label_type: Name of the label of the data pairs
      skip_first_line: If True, first line of data file will be ignored
      separator: Separator between columns in the data file
      encoding: Encoding of the data file
      label: If False, the dataset expects unlabeled data
docstr_tok: ["Creates","a","Dataset","for","pairs","of","sentences\/paragraphs",".","The","file","needs","to","be","in","a","column","format",",","where","each","line","has","a","column","for","the","first","sentence\/paragraph",",","the","second","sentence\/paragraph","and","the","label","seperated","by","e.g",".","'\\t","'","(","just","like","in","the","glue","RTE-dataset","https",":","\/\/gluebenchmark.com\/tasks",")",".","For","each","data","pair","we","create","a","flair.data.DataPair","object",".","Args",":","path_to_data",":","path","to","the","data","file","columns",":","list","of","integers","that","indicate","the","respective","columns",".","The","first","entry","is","the","column","for","the","first","sentence",",","the","second","for","the","second","sentence","and","the","third","for","the","label",".","Default","[","0,1,2","]","max_tokens_per_doc",":","If","set",",","shortens","sentences","to","this","maximum","number","of","tokens","max_chars_per_doc",":","If","set",",","shortens","sentences","to","this","maximum","number","of","characters","use_tokenizer",":","Whether","to","use","in-built","tokenizer","in_memory",":","If","True",",","data","will","be","saved","in","list","of","flair.data.DataPair","objects",",","otherwise","we","use","lists","with","simple","strings","which","needs","less","space","label_type",":","Name","of","the","label","of","the","data","pairs","skip_first_line",":","If","True",",","first","line","of","data","file","will","be","ignored","separator",":","Separator","between","columns","in","the","data","file","encoding",":","Encoding","of","the","data","file","label",":","If","False",",","the","dataset","expects","unlabeled","data"]
code: def __init__(
      self,
      path_to_data: Union[str, Path],
      columns: List[int] = [0, 1, 2],
      max_tokens_per_doc=-1,
      max_chars_per_doc=-1,
      use_tokenizer=True,
      in_memory: bool = True,
      label_type: Optional[str] = None,
      skip_first_line: bool = False,
      separator: str = "\t",
      encoding: str = "utf-8",
      label: bool = True,
  ) -> None:
      r"""Creates a Dataset for pairs of sentences/paragraphs.
  
      The file needs to be in a column format,
      where each line has a column for the first sentence/paragraph, the second sentence/paragraph and the label
      seperated by e.g. '\t' (just like in the glue RTE-dataset https://gluebenchmark.com/tasks) .
      For each data pair we create a flair.data.DataPair object.
  
      Args:
          path_to_data: path to the data file
          columns: list of integers that indicate the respective columns. The first entry is the column for the first sentence, the second for the second sentence and the third for the label. Default [0,1,2]
          max_tokens_per_doc: If set, shortens sentences to this maximum number of tokens
          max_chars_per_doc: If set, shortens sentences to this maximum number of characters
          use_tokenizer: Whether to use in-built tokenizer
          in_memory: If True, data will be saved in list of flair.data.DataPair objects, otherwise we use lists with simple strings which needs less space
          label_type: Name of the label of the data pairs
          skip_first_line: If True, first line of data file will be ignored
          separator: Separator between columns in the data file
          encoding: Encoding of the data file
          label: If False, the dataset expects unlabeled data
      """
      path_to_data = Path(path_to_data)
  
      # stop if file does not exist
      assert path_to_data.exists()
  
      self.in_memory = in_memory
  
      self.use_tokenizer = use_tokenizer
  
      self.max_tokens_per_doc = max_tokens_per_doc
  
      self.label = label
  
      assert label_type is not None
      self.label_type = label_type
  
      self.total_data_count: int = 0
  
      if self.in_memory:
          self.data_pairs: List[DataPair] = []
      else:
          self.first_elements: List[str] = []
          self.second_elements: List[str] = []
          self.labels: List[Optional[str]] = []
  
      with open(str(path_to_data), encoding=encoding) as source_file:
          source_line = source_file.readline()
  
          if skip_first_line:
              source_line = source_file.readline()
  
          while source_line:
              source_line_list = source_line.strip().split(separator)
  
              first_element = source_line_list[columns[0]]
              second_element = source_line_list[columns[1]]
  
              if self.label:
                  pair_label: Optional[str] = source_line_list[
                      columns[2]
                  ]
              else:
                  pair_label = None
  
              if max_chars_per_doc > 0:
                  first_element = first_element[:max_chars_per_doc]
                  second_element = second_element[:max_chars_per_doc]
  
              if self.in_memory:
                  data_pair = self._make_data_pair(
                      first_element, second_element, pair_label
                  )
                  self.data_pairs.append(data_pair)
              else:
                  self.first_elements.append(first_element)
                  self.second_elements.append(second_element)
                  if self.label:
                      self.labels.append(pair_label)
  
              self.total_data_count += 1
  
              source_line = source_file.readline()
  
imports: import logging
  import os
  from pathlib import Path
  from typing import List, Optional, Union
  import flair
  from flair.data import Corpus, DataPair, FlairDataset, Sentence, TextPair, _iter_dataset
  from flair.datasets.base import find_train_dev_test_files
  from flair.file_utils import cached_path, unpack_file, unzip_file
label_desc: Use image node_id 1 to create a new DataPairDataset object from inherited base classes: FlairDataset with example: obj = DataPairDataset(path_to_data, columns, max_tokens_per_doc, max_chars_per_doc, use_tokenizer, in_memory, label_type, skip_first_line, separator, encoding, label)