GF_ISMASample *gf_isom_ismacryp_sample_from_data(u8 *data, u32 dataLength, Bool use_selective_encryption, u8 KI_length, u8 IV_length)
{
	GF_ISMASample *s;
	GF_BitStream *bs;
	/*empty text sample*/
	if (!data || !dataLength) {
		return gf_isom_ismacryp_new_sample();
	}

	s = gf_isom_ismacryp_new_sample();

	/*empty sample*/
	if (!data || !dataLength) return s;

	bs = gf_bs_new(data, dataLength, GF_BITSTREAM_READ);

	s->dataLength = dataLength;
	s->IV_length = IV_length;
	s->KI_length = KI_length;

	if (use_selective_encryption) {
		s->flags = GF_ISOM_ISMA_USE_SEL_ENC;
		if (s->dataLength < 1) goto exit;
		if (gf_bs_read_int(bs, 1)) s->flags |= GF_ISOM_ISMA_IS_ENCRYPTED;
		gf_bs_read_int(bs, 7);
		s->dataLength -= 1;
	} else {
		s->flags = GF_ISOM_ISMA_IS_ENCRYPTED;
	}
	if (s->flags & GF_ISOM_ISMA_IS_ENCRYPTED) {
		if (IV_length != 0) {
			if (s->dataLength < IV_length) goto exit;
			s->IV = gf_bs_read_long_int(bs, 8*IV_length);
			s->dataLength -= IV_length;
		}
		if (KI_length) {
			if (s->dataLength < KI_length) goto exit;
			s->key_indicator = (u8 *)gf_malloc(KI_length);
			if (!s->key_indicator) goto exit;
			gf_bs_read_data(bs, (char*)s->key_indicator, KI_length);
			s->dataLength -= KI_length;
		}
	}
	s->data = (char*)gf_malloc(sizeof(char)*s->dataLength);
	if (!s->data) goto exit;
	gf_bs_read_data(bs, s->data, s->dataLength);
	gf_bs_del(bs);
	return s;

exit:
	gf_isom_ismacryp_delete_sample(s);
	return NULL;
}

fixed #1735


static GF_Err isom_set_protected_entry(GF_ISOFile *the_file, u32 trackNumber, u32 desc_index, u8 version, u32 flags,
        u32 scheme_type, u32 scheme_version, char *scheme_uri, Bool is_isma, GF_ProtectionSchemeInfoBox **out_sinf)
{
	u32 original_format;
	GF_Err e;
	GF_SampleEntryBox *sea;
	GF_ProtectionSchemeInfoBox *sinf;
	GF_TrackBox *trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak) return GF_BAD_PARAM;

	e = Media_GetSampleDesc(trak->Media, desc_index, &sea, NULL);
	if (e) return e;

	original_format = sea->type;
	if (original_format==GF_ISOM_BOX_TYPE_GNRA) {
		original_format = ((GF_GenericAudioSampleEntryBox*)sea)->EntryType;
	} else if (original_format==GF_ISOM_BOX_TYPE_GNRV) {
		original_format = ((GF_GenericVisualSampleEntryBox*)sea)->EntryType;
	} else if (original_format==GF_ISOM_BOX_TYPE_GNRM) {
		original_format = ((GF_GenericSampleEntryBox*)sea)->EntryType;
	}

	/* Replacing the Media Type */
	switch (sea->type) {
	case GF_ISOM_BOX_TYPE_MP4A:
	case GF_ISOM_BOX_TYPE_DAMR:
	case GF_ISOM_BOX_TYPE_DEVC:
	case GF_ISOM_BOX_TYPE_DQCP:
	case GF_ISOM_BOX_TYPE_DSMV:
	case GF_ISOM_BOX_TYPE_AC3:
	case GF_ISOM_BOX_TYPE_EC3:
		sea->type = GF_ISOM_BOX_TYPE_ENCA;
		break;
	case GF_ISOM_BOX_TYPE_MP4V:
	case GF_ISOM_BOX_TYPE_D263:
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	/*special case for AVC1*/
	case GF_ISOM_BOX_TYPE_AVC1:
	case GF_ISOM_BOX_TYPE_AVC2:
	case GF_ISOM_BOX_TYPE_AVC3:
	case GF_ISOM_BOX_TYPE_AVC4:
	case GF_ISOM_BOX_TYPE_SVC1:
	case GF_ISOM_BOX_TYPE_MVC1:
		if (is_isma)
			original_format = GF_ISOM_BOX_TYPE_264B;
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	case GF_ISOM_BOX_TYPE_HVC1:
	case GF_ISOM_BOX_TYPE_HEV1:
	case GF_ISOM_BOX_TYPE_HVC2:
	case GF_ISOM_BOX_TYPE_HEV2:
	case GF_ISOM_BOX_TYPE_LHE1:
	case GF_ISOM_BOX_TYPE_LHV1:
	case GF_ISOM_BOX_TYPE_HVT1:
		if (is_isma)
			original_format = GF_ISOM_BOX_TYPE_265B;
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	case GF_ISOM_BOX_TYPE_VVC1:
	case GF_ISOM_BOX_TYPE_VVI1:
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	case GF_ISOM_BOX_TYPE_VP09:
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	case GF_ISOM_BOX_TYPE_AV01:
		sea->type = GF_ISOM_BOX_TYPE_ENCV;
		break;
	case GF_ISOM_BOX_TYPE_MP4S:
	case GF_ISOM_BOX_TYPE_LSR1:
		sea->type = GF_ISOM_BOX_TYPE_ENCS;
		break;
	case GF_ISOM_BOX_TYPE_STXT:
	case GF_ISOM_BOX_TYPE_WVTT:
	case GF_ISOM_BOX_TYPE_STPP:
		sea->type = GF_ISOM_BOX_TYPE_ENCT;
		break;
	case GF_ISOM_BOX_TYPE_ENCA:
	case GF_ISOM_BOX_TYPE_ENCV:
	case GF_ISOM_BOX_TYPE_ENCT:
	case GF_ISOM_BOX_TYPE_ENCM:
	case GF_ISOM_BOX_TYPE_ENCF:
	case GF_ISOM_BOX_TYPE_ENCS:
		GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] cannot set protection entry: file is already encrypted.\n"));
		return GF_BAD_PARAM;
	default:
		if (sea->internal_type == GF_ISOM_SAMPLE_ENTRY_AUDIO) {
			sea->type = GF_ISOM_BOX_TYPE_ENCA;
		} else if (sea->internal_type == GF_ISOM_SAMPLE_ENTRY_VIDEO) {
			sea->type = GF_ISOM_BOX_TYPE_ENCV;
		} else {
			GF_LOG(GF_LOG_WARNING, GF_LOG_CONTAINER, ("[iso file] unsupported entry type %s for CENC.\n", gf_4cc_to_str(sea->type) ));
			return GF_BAD_PARAM;
		}
	}

	sinf = (GF_ProtectionSchemeInfoBox *)gf_isom_box_new_parent(&sea->child_boxes, GF_ISOM_BOX_TYPE_SINF);
	if (!sinf) return GF_OUT_OF_MEM;

	sinf->scheme_type = (GF_SchemeTypeBox *)gf_isom_box_new_parent(&sinf->child_boxes, GF_ISOM_BOX_TYPE_SCHM);
	if (!sinf->scheme_type) return GF_OUT_OF_MEM;

	sinf->scheme_type->version = version;
	sinf->scheme_type->flags = flags;
	sinf->scheme_type->scheme_type = scheme_type;
	sinf->scheme_type->scheme_version = scheme_version;
	if (scheme_uri && (sinf->scheme_type->flags == 1)) {
		sinf->scheme_type->URI = (char *)gf_malloc(sizeof(char)*strlen(scheme_uri));
		if (!sinf->scheme_type->URI) return GF_OUT_OF_MEM;
		memmove(sinf->scheme_type->URI, scheme_uri, strlen(scheme_uri));
	}

	sinf->original_format = (GF_OriginalFormatBox *)gf_isom_box_new_parent(&sinf->child_boxes, GF_ISOM_BOX_TYPE_FRMA);
	if (!sinf->original_format) return GF_OUT_OF_MEM;
	sinf->original_format->data_format = original_format;

	//common to isma, cenc and oma
	sinf->info = (GF_SchemeInformationBox *)gf_isom_box_new_parent(&sinf->child_boxes, GF_ISOM_BOX_TYPE_SCHI);

	*out_sinf = sinf;
	return GF_OK;
}

fixed #1735


Bool gf_isom_is_cenc_media(GF_ISOFile *the_file, u32 trackNumber, u32 sampleDescriptionIndex)
{
	GF_TrackBox *trak;
	GF_ProtectionSchemeInfoBox *sinf;
	u32 i, count;

	trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak) return GF_FALSE;

	count = gf_list_count(trak->Media->information->sampleTable->SampleDescription->child_boxes);
	for (i=0; i<count; i++) {
		if (sampleDescriptionIndex && (i+1 != sampleDescriptionIndex)) continue;

		sinf = isom_get_sinf_entry(trak, i+1, GF_ISOM_CENC_SCHEME, NULL);
		if (!sinf) sinf = isom_get_sinf_entry(trak, i+1, GF_ISOM_CBC_SCHEME, NULL);
		if (!sinf) sinf = isom_get_sinf_entry(trak, i+1, GF_ISOM_CENS_SCHEME, NULL);
		if (!sinf) sinf = isom_get_sinf_entry(trak, i+1, GF_ISOM_CBCS_SCHEME, NULL);
		if (!sinf) sinf = isom_get_sinf_entry(trak, i+1, GF_ISOM_PIFF_SCHEME, NULL);

		if (!sinf) continue;

		/*non-encrypted or non-CENC*/
		if (!sinf->scheme_type)
			return GF_FALSE;

		switch (sinf->scheme_type->scheme_type) {
		case GF_ISOM_CENC_SCHEME:
		case GF_ISOM_CBC_SCHEME:
		case GF_ISOM_CENS_SCHEME:
		case GF_ISOM_CBCS_SCHEME:
		case GF_ISOM_SVE1_SCHEME:
			return GF_TRUE;
		default:
			return GF_FALSE;
		}
		return GF_TRUE;
	}
	return GF_FALSE;

}

fixed #1735


GF_SampleEncryptionBox * gf_isom_create_piff_psec_box(u8 version, u32 flags, u32 AlgorithmID, u8 IV_size, bin128 KID)
{
	GF_SampleEncryptionBox *psec;

	psec = (GF_SampleEncryptionBox *) gf_isom_box_new(GF_ISOM_BOX_UUID_PSEC);
	if (!psec)
		return NULL;
	psec->version = version;
	psec->flags = flags;
	psec->piff_type = 1;
	if (psec->flags & 0x1) {
		psec->AlgorithmID = AlgorithmID;
		psec->IV_size = IV_size;
		strcpy((char *)psec->KID, (const char *)KID);
	}
	psec->samp_aux_info = gf_list_new();

	return psec;
}

fixed #1735


GF_Err gf_isom_piff_allocate_storage(GF_ISOFile *the_file, u32 trackNumber, u32 AlgorithmID, u8 IV_size, bin128 KID)
{
	GF_TrackBox *trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak) return GF_BAD_PARAM;

	if (trak->sample_encryption) return GF_OK;
	trak->sample_encryption = (GF_SampleEncryptionBox *)gf_isom_create_piff_psec_box(1, 0, AlgorithmID, IV_size, KID);
	//senc will be written and destroyed with the other boxes
	if (!trak->child_boxes) trak->child_boxes = gf_list_new();
	return gf_list_add(trak->child_boxes, trak->sample_encryption);
}

fixed #1735


GF_SampleEncryptionBox * gf_isom_create_samp_enc_box(u8 version, u32 flags)
{
	GF_SampleEncryptionBox *senc;

	senc = (GF_SampleEncryptionBox *) gf_isom_box_new(GF_ISOM_BOX_TYPE_SENC);
	if (!senc)
		return NULL;
	senc->version = version;
	senc->flags = flags;
	senc->samp_aux_info = gf_list_new();

	return senc;
}

fixed #1735


void gf_isom_cenc_set_saiz_saio(GF_SampleEncryptionBox *senc, GF_SampleTableBox *stbl, GF_TrackFragmentBox  *traf, u32 len, Bool saio_32bits, Bool use_multikey)
{
	u32  i;
	GF_List **child_boxes = stbl ? &stbl->child_boxes : &traf->child_boxes;
	if (!senc->cenc_saiz) {
		senc->cenc_saiz = (GF_SampleAuxiliaryInfoSizeBox *) gf_isom_box_new_parent(child_boxes, GF_ISOM_BOX_TYPE_SAIZ);
		if (!senc->cenc_saiz) return;
		//as per 3rd edition of cenc "so content SHOULD be created omitting these optional fields" ...
		senc->cenc_saiz->aux_info_type = 0;
		senc->cenc_saiz->aux_info_type_parameter = use_multikey ? 1 : 0;
		if (stbl)
			stbl_on_child_box((GF_Box*)stbl, (GF_Box *)senc->cenc_saiz, GF_FALSE);
		else
			traf_on_child_box((GF_Box*)traf, (GF_Box *)senc->cenc_saiz, GF_FALSE);
	}
	if (!senc->cenc_saio) {
		senc->cenc_saio = (GF_SampleAuxiliaryInfoOffsetBox *) gf_isom_box_new_parent(child_boxes, GF_ISOM_BOX_TYPE_SAIO);
		if (!senc->cenc_saio) return;
		//force using version 1 for saio box, it could be redundant when we use 64 bits for offset
		senc->cenc_saio->version = saio_32bits ? 0 : 1;
		//as per 3rd edition of cenc "so content SHOULD be created omitting these optional fields" ...
		senc->cenc_saio->aux_info_type = 0;
		senc->cenc_saiz->aux_info_type_parameter = use_multikey ? 1 : 0;
		senc->cenc_saio->entry_count = 1;
		if (stbl)
			stbl_on_child_box((GF_Box*)stbl, (GF_Box *)senc->cenc_saio, GF_FALSE);
		else
			traf_on_child_box((GF_Box*)traf, (GF_Box *)senc->cenc_saio, GF_FALSE);
	}

	if (!senc->cenc_saiz->sample_count || ((senc->cenc_saiz->default_sample_info_size==len) && len) ) {
		senc->cenc_saiz->sample_count ++;
		senc->cenc_saiz->default_sample_info_size = len;
	} else {
		if (senc->cenc_saiz->sample_count + 1 > senc->cenc_saiz->sample_alloc) {
			if (!senc->cenc_saiz->sample_alloc) senc->cenc_saiz->sample_alloc = senc->cenc_saiz->sample_count+1;
			else senc->cenc_saiz->sample_alloc *= 2;

			senc->cenc_saiz->sample_info_size = (u8*)gf_realloc(senc->cenc_saiz->sample_info_size, sizeof(u8)*(senc->cenc_saiz->sample_alloc));
		}

		if (senc->cenc_saiz->default_sample_info_size || (senc->cenc_saiz->sample_count==1)) {
			for (i=0; i<senc->cenc_saiz->sample_count; i++)
				senc->cenc_saiz->sample_info_size[i] = senc->cenc_saiz->default_sample_info_size;
			senc->cenc_saiz->default_sample_info_size = 0;
		}
		senc->cenc_saiz->sample_info_size[senc->cenc_saiz->sample_count] = len;
		senc->cenc_saiz->sample_count++;
	}
}

fixed #1735


GF_Err gf_isom_change_ismacryp_protection(GF_ISOFile *the_file, u32 trackNumber, u32 sampleDescriptionIndex, char *scheme_uri, char *kms_uri)
{
	GF_TrackBox *trak;
	GF_Err e;
	GF_SampleEntryBox *sea;
	GF_ProtectionSchemeInfoBox *sinf;

	e = CanAccessMovie(the_file, GF_ISOM_OPEN_WRITE);
	if (e) return e;

	trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak || !trak->Media || !sampleDescriptionIndex) return GF_BAD_PARAM;

	sea = NULL;
	sinf = isom_get_sinf_entry(trak, sampleDescriptionIndex, GF_ISOM_ISMACRYP_SCHEME, &sea);
	if (!sinf) return GF_OK;

	if (scheme_uri) {
		gf_free(sinf->scheme_type->URI);
		sinf->scheme_type->URI = gf_strdup(scheme_uri);
	}
	if (kms_uri) {
		gf_free(sinf->info->ikms->URI);
		sinf->info->ikms->URI = gf_strdup(kms_uri);
	}
	return GF_OK;
}

fixed #1735


void gf_isom_ismacryp_delete_sample(GF_ISMASample *samp)
{
	if (!samp) return;
	if (samp->data && samp->dataLength) gf_free(samp->data);
	if (samp->key_indicator) gf_free(samp->key_indicator);
	gf_free(samp);
}

fixed #1735


inline bool SparseTensor::ValidateAndInitializeToDense(Tensor* out,
                                                       bool initialize) {
  DCHECK_EQ(DataTypeToEnum<T>::v(), dtype())
      << "ToDense requested with the wrong datatype";

  DCHECK_EQ(out->shape().dims(), dims_)
      << "Incompatible dimensions between SparseTensor and output";

  DCHECK_EQ(out->dtype(), DataTypeToEnum<T>::v())
      << "Output must be type: " << DataTypeToEnum<T>::v()
      << " but got: " << out->dtype();

  // Make sure the dense output is the same rank and has room
  // to hold the SparseTensor.
  const auto& out_shape = out->shape();
  if (shape_.size() != out_shape.dims()) return false;
  for (int d = 0; d < shape_.size(); ++d) {
    if (shape_[d] > out_shape.dim_size(d)) return false;
  }

  if (initialize) {
    auto out_t = out->flat<T>();
    out_t.setConstant(T());
  }

  return true;
}

Fix heap-buffer-overflow issue with `tf.raw_ops.SparseSplit`.

PiperOrigin-RevId: 371242872
Change-Id: I482bb3d12602c7c3cc9446f97fb9f584bb98e9a4


inline SparseTensor SparseTensor::Concat(
    const gtl::ArraySlice<SparseTensor>& tensors) {
  DCHECK_GE(tensors.size(), size_t{1}) << "Cannot concat 0 SparseTensors";
  const int dims = tensors[0].dims_;
  DCHECK_GE(dims, 1) << "Cannot concat 0-dimensional SparseTensors";
  auto order_0 = tensors[0].order();
  const int primary_dim = order_0[0];
  ShapeArray final_order(order_0.begin(), order_0.end());
  ShapeArray final_shape(tensors[0].shape().begin(), tensors[0].shape().end());
  final_shape[primary_dim] = 0;  // We'll build this up as we go along.
  int num_entries = 0;

  bool fully_ordered = true;
  for (const SparseTensor& st : tensors) {
    DCHECK_EQ(st.dims_, dims) << "All SparseTensors must have the same rank.";
    DCHECK_EQ(DataTypeToEnum<T>::v(), st.dtype())
        << "Concat requested with the wrong data type";
    DCHECK_GE(st.order()[0], 0) << "SparseTensor must be ordered";
    DCHECK_EQ(st.order()[0], primary_dim)
        << "All SparseTensors' order[0] must match.  This is the concat dim.";
    if (st.order() != final_order) fully_ordered = false;
    const VarDimArray& st_shape = st.shape();
    for (int d = 0; d < dims - 1; ++d) {
      const int cdim = (d < primary_dim) ? d : d + 1;
      DCHECK_EQ(final_shape[cdim], st_shape[cdim])
          << "All SparseTensors' shapes must match except on the concat dim.  "
          << "Concat dim: " << primary_dim
          << ", mismatched shape at dim: " << cdim
          << ".  Expecting shape like: [" << str_util::Join(final_shape, ",")
          << "] but saw shape: [" << str_util::Join(st_shape, ",") << "]";
    }

    // Update dimension of final shape
    final_shape[primary_dim] =
        (final_shape[primary_dim] + st_shape[primary_dim]);

    num_entries += st.num_entries();  // Update number of entries
  }

  // If nonconsistent ordering among inputs, set final order to -1s.
  if (!fully_ordered) {
    final_order = UndefinedOrder(final_shape);
  }

  Tensor output_ix(DT_INT64, TensorShape({num_entries, dims}));
  Tensor output_vals(DataTypeToEnum<T>::v(), TensorShape({num_entries}));

  TTypes<int64>::Matrix ix_t = output_ix.matrix<int64>();
  typename TTypes<T>::Vec vals_t = output_vals.vec<T>();

  Eigen::DenseIndex offset = 0;
  int64 shape_offset = 0;
  for (const SparseTensor& st : tensors) {
    const int st_num_entries = st.num_entries();

    // Fill in indices & values.
    if (st_num_entries > 0) {
      std::copy_n(&st.vals_.vec<T>()(0), st_num_entries, &vals_t(offset));

      const auto* st_ix = &st.ix_.matrix<int64>()(0, 0);
      auto* ix_out = &ix_t(offset, 0);
      for (std::size_t i = 0; i < st_num_entries * dims; ++i) {
        *ix_out++ = *st_ix++ + ((i % dims == primary_dim) ? shape_offset : 0);
      }
    }

    offset += st_num_entries;
    shape_offset += st.shape()[primary_dim];
  }

  return SparseTensor(output_ix, output_vals, final_shape, final_order);
}


inline bool SparseTensor::ToDense(Tensor* out, bool initialize) {
  if (!ValidateAndInitializeToDense<T>(out, initialize)) return false;

  auto out_t = out->flat<T>();
  auto vals_t = vals_.vec<T>();
  auto ix_t = ix_.matrix<int64>();
  const int64* const ix_ptr = ix_t.data();

  if (dims_ == 1) {
    // Fast path for sparse vectors.
    const int64 out_length = out->shape().dim_size(0);
    for (int n = 0; n < vals_t.dimension(0); ++n) {
      const int64 index = internal::SubtleMustCopy(ix_ptr[n]);
      if (!FastBoundsCheck(index, out_length)) return false;
      out_t(index) = vals_t(n);
    }
    return true;
  } else if (dims_ == 2) {
    // Fast path for sparse matrices.
    const auto& out_shape = out->shape();
    const int64 out_rows = out_shape.dim_size(0);
    const int64 out_cols = out_shape.dim_size(1);
    for (int n = 0; n < vals_t.dimension(0); ++n) {
      const int64 row_index = internal::SubtleMustCopy(ix_ptr[n * 2]);
      const int64 col_index = internal::SubtleMustCopy(ix_ptr[n * 2 + 1]);
      if (!(FastBoundsCheck(row_index, out_rows) &&
            FastBoundsCheck(col_index, out_cols))) {
        return false;
      }
      out_t(row_index * out_cols + col_index) = vals_t(n);
    }
    return true;
  } else {
    // General path for N-dimensional sparse tensors.
    gtl::InlinedVector<int64, 4> strides(dims_);
    const auto& out_shape = out->shape().dim_sizes();
    if (dims_ > 0) {
      strides[dims_ - 1] = 1;
    }
    for (int d = dims_ - 2; d >= 0; --d) {
      strides[d] = strides[d + 1] * out_shape[d + 1];
    }

    for (int n = 0; n < vals_t.dimension(0); ++n) {
      bool invalid_dims = false;
      int64 ix = 0;
      for (int d = 0; d < dims_; ++d) {
        const int64 ix_n_d = internal::SubtleMustCopy(ix_ptr[n * dims_ + d]);
        if (!FastBoundsCheck(ix_n_d, out_shape[d])) {
          invalid_dims = true;
        }
        ix += strides[d] * ix_n_d;
      }
      if (invalid_dims) return false;
      out_t(ix) = vals_t(n);
    }
    return true;
  }
}


int dims() const { return shape_.size(); }


VarDimArray shape() const { return shape_; }


  static inline int GetDimensionInSlice(const int dim, const int split_size,
                                        const int residual) {
    DCHECK_GT(split_size, 0);
    DCHECK_GE(dim, 0);
    if (residual == 0) return dim % split_size;
    const int offset = residual * (split_size + 1);
    if (dim < offset) {
      return dim % (split_size + 1);
    } else {
      return (dim - offset) % split_size;
    }
  }


  static inline ShapeArray TensorShapeToVector(const TensorShape& shape) {
    ShapeArray vec(shape.dims());
    for (int i = 0; i < shape.dims(); ++i) vec[i] = shape.dim_size(i);
    return vec;
  }


inline SparseTensor SparseTensor::Slice(const SparseTensor& input_tensor,
                                        const gtl::ArraySlice<int64>& start,
                                        const gtl::ArraySlice<int64>& size) {
  TensorShape output_shape(input_tensor.shape());

  const int dims = input_tensor.dims();
  for (int dim = 0; dim < dims; dim++) {
    // Determine the size of the result; if the selected slice goes beyond the
    // input boundary, the result will correspond to the size of the overlap
    // between the input and the selected slice.
    const int64 input_size = output_shape.dim_size(dim);
    const int64 start_index = start[dim];
    const int64 slice_size = size[dim];
    if (start_index + slice_size < input_size) {
      // The entire selection is within input boundaries.
      output_shape.set_dim(dim, slice_size);
    } else if (start_index < input_size) {
      // The selection starts within input boundaries, but goes beyond them.
      output_shape.set_dim(dim, input_size - start_index);
    } else {
      // The selection is entirely out of input boundaries.
      output_shape.set_dim(dim, 0);
    }
  }

  auto input_indices_t = input_tensor.indices().matrix<int64>();
  auto input_values_t = input_tensor.values().vec<T>();

  // Find the number of indices that fall inside start and size.
  int count = 0;
  for (int i = 0; i < input_tensor.indices().dim_size(0); i++) {
    // The following will check to see if an input is within the
    // range specified by start and size.
    // The for loop below iterates through all dimensions. In case
    // the index falls outside of the start and size at any dimension,
    // it will be considered as a "no hit" (hit = false). In this
    // case, it will not be counted as the index that fall inside
    // the range specified by start and size.
    bool hit = true;
    for (int dim = 0; dim < dims; dim++) {
      if (!(start[dim] <= input_indices_t(i, dim) &&
            input_indices_t(i, dim) < start[dim] + size[dim])) {
        hit = false;
        break;
      }
    }
    if (!hit) {
      continue;
    }
    count++;
  }

  Tensor output_values(DataTypeToEnum<T>::v(), TensorShape({count}));
  Tensor output_indices(DT_INT64, TensorShape({count, dims}));

  auto output_values_t = output_values.vec<T>();
  auto output_indices_t = output_indices.matrix<int64>();

  // Obtain the output indices that fall inside start and size.
  int index = 0;
  for (int i = 0; i < input_tensor.indices().dim_size(0) && index < count;
       i++) {
    // The logic here is similar as the above except that the above
    // only count the number of indices while here we actually generate
    // the output.
    bool hit = true;
    for (int dim = 0; dim < dims; dim++) {
      if (!(start[dim] <= input_indices_t(i, dim) &&
            input_indices_t(i, dim) < start[dim] + size[dim])) {
        hit = false;
        break;
      }
    }
    if (!hit) {
      continue;
    }
    output_values_t(index) = input_values_t(i);
    for (int dim = 0; dim < dims; dim++) {
      output_indices_t(index, dim) = input_indices_t(i, dim) - start[dim];
    }
    index++;
  }

  return SparseTensor(output_indices, output_values, output_shape);
}


DataType dtype() const { return vals_.dtype(); }


  SparseTensor(const SparseTensor& other)
      : SparseTensor(other.ix_, other.vals_, other.shape_, other.order_) {}


  SparseTensor& operator=(SparseTensor&& other) {
    ix_ = std::move(other.ix_);
    vals_ = std::move(other.vals_);
    shape_ = std::move(other.shape_);
    order_ = std::move(other.order_);
    dims_ = std::move(other.dims_);
    return *this;
  }


VarDimArray order() const { return order_; }


std::size_t num_entries() const { return ix_.dim_size(0); }


  SparseTensor(SparseTensor&& other)
      : SparseTensor(std::move(other.ix_), std::move(other.vals_),
                     std::move(other.shape_), std::move(other.order_)) {}


  SparseTensor& operator=(const SparseTensor& other) {
    ix_ = other.ix_;
    vals_ = other.vals_;
    shape_ = other.shape_;
    order_ = other.order_;
    dims_ = other.dims_;
    return *this;
  }


const Tensor& indices() const { return ix_; }


  SparseTensor(Tensor ix, Tensor vals, const TensorShape& shape)
      : SparseTensor(std::move(ix), std::move(vals), TensorShapeToVector(shape),
                     UndefinedOrder(TensorShapeToVector(shape))) {}


  std::vector<int64> PickDims(gtl::ArraySlice<int64> dim_indices) const {
    std::vector<int64> res(dim_indices.size());
    for (size_t i = 0; i < dim_indices.size(); ++i) {
      res[i] = shape_[dim_indices[i]];
    }
    return res;
  }


const Tensor& values() const { return vals_; }


  GroupIterable group(const VarDimArray& group_ix) const {
    DCHECK_LE(group_ix.size(), dims_);
    for (std::size_t di = 0; di < group_ix.size(); ++di) {
      DCHECK_GE(group_ix[di], 0) << "Group dimension out of range";
      DCHECK_LT(group_ix[di], dims_) << "Group dimension out of range";
      DCHECK_EQ(group_ix[di], order_[di])
          << "Group dimension does not match sorted order";
    }
    return GroupIterable(ix_, vals_, dims_, group_ix);
  }


  static inline ShapeArray UndefinedOrder(const VarDimArray shape) {
    return ShapeArray(shape.size(), -1);
  }


inline void SparseTensor::Reorder(const VarDimArray& order) {
  DCHECK_EQ(DataTypeToEnum<T>::v(), dtype())
      << "Reorder requested with the wrong datatype";
  DCHECK_EQ(order.size(), dims_) << "Order length must be SparseTensor rank";
  auto ix_t = ix_.matrix<int64>();
  auto vals_t = vals_.vec<T>();

  std::vector<int64> reorder(num_entries());
  std::iota(reorder.begin(), reorder.end(), 0);

  // Sort to get order of indices
  switch (order.size()) {
#define CASE_SORT(ORDER_SIZE)                                    \
  case ORDER_SIZE: {                                             \
    FixedDimComparator<ORDER_SIZE> sorter(ix_t, order, shape()); \
    std::sort(reorder.begin(), reorder.end(), sorter);           \
    break;                                                       \
  }
    CASE_SORT(0);
    CASE_SORT(1);
    CASE_SORT(2);
    CASE_SORT(3);
    CASE_SORT(4);
    CASE_SORT(5);
#undef CASE_SORT
    default: {
      DimComparator sorter(ix_t, order, shape());
      std::sort(reorder.begin(), reorder.end(), sorter);
    }
  }

  // We have a forward reordering, but what we'll need is a
  // permutation (the inverse).  This can be calculated with O(1)
  // additional
  // and O(n) time (INVPERM) but we just do the simple thing here.
  std::vector<size_t> permutation(reorder.size());
  for (std::size_t n = 0; n < reorder.size(); ++n) {
    permutation[reorder[n]] = n;
  }

  // Update indices & values by converting the permutations to
  // a product of transpositions.  Iterate over the cycles in the
  // permutation, and convert each of those into a product of
  // transpositions (swaps):
  //   https://en.wikipedia.org/wiki/Cyclic_permutation
  // This is N swaps, 2*N comparisons.
  for (std::size_t n = 0; n + 1 < permutation.size(); ++n) {
    while (n != permutation[n]) {
      std::size_t r = permutation[n];
      std::swap_ranges(&(ix_t(n, 0)), &(ix_t(n + 1, 0)), &(ix_t(r, 0)));
      std::swap(vals_t(n), vals_t(r));
      std::swap(permutation[n], permutation[r]);
    }
  }

  order_ = ShapeArray(order.begin(), order.end());
}


  SparseTensor(Tensor ix, Tensor vals, const TensorShape& shape,
               const VarDimArray order)
      : SparseTensor(std::move(ix), std::move(vals), TensorShapeToVector(shape),
                     order) {}


inline Status SparseTensor::Split(const SparseTensor& input_tensor,
                                  const int split_dim, const int num_split,
                                  std::vector<SparseTensor>* result) {
  std::vector<Tensor> output_indices;
  std::vector<Tensor> output_values;
  std::vector<TensorShape> output_shapes;
  output_indices.reserve(num_split);
  output_values.reserve(num_split);
  output_shapes.reserve(num_split);

  std::vector<typename TTypes<int64>::Matrix> output_indices_t;
  std::vector<typename TTypes<T>::Vec> output_values_t;
  output_indices_t.reserve(num_split);
  output_values_t.reserve(num_split);
  auto input_values_t = input_tensor.values().vec<T>();
  auto input_indices_t = input_tensor.indices().matrix<int64>();

  std::vector<int> num_values(num_split, 0);
  const int num_dim = input_tensor.shape().size();
  const int split_dim_size = input_tensor.shape()[split_dim];
  const int split_size = split_dim_size / num_split;

  if (!(num_split > 0 && num_split <= split_dim_size)) {
    return errors::InvalidArgument("num_split must be in the interval (0, ",
                                   split_dim_size, "]");
  }
  if (!(split_dim >= 0 && split_dim < num_dim)) {
    return errors::InvalidArgument("num_dim must be in the interval [0, ",
                                   num_dim, ")");
  }

  const int residual = split_dim_size % num_split;
  for (int i = 0; i < input_tensor.indices().dim_size(0); ++i) {
    const int dim = input_tensor.indices().matrix<int64>()(i, split_dim);
    int slice_index = GetSliceIndex(dim, split_size, residual);
    if (slice_index >= num_values.size()) {
      return errors::InvalidArgument("Slice index ", slice_index,
                                     " is larger than num_split.");
    }
    num_values[slice_index]++;
  }

  for (int i = 0; i < num_split; ++i) {
    // TODO(ataei): Pass an allocator to avoid allocating large memory buffer.
    output_indices.emplace_back(DT_INT64,
                                TensorShape({num_values[i], num_dim}));
    output_values.emplace_back(DataTypeToEnum<T>::v(),
                               TensorShape({num_values[i]}));
    output_shapes.emplace_back(input_tensor.shape());
    output_indices_t.emplace_back(output_indices[i].matrix<int64>());
    output_values_t.emplace_back(output_values[i].vec<T>());
    const int size = GetSliceShape(i, split_size, residual);
    output_shapes[i].set_dim(split_dim, size);
  }

  std::vector<int> values_inserted_in_slice(num_split, 0);
  for (int i = 0; i < input_tensor.indices().dim_size(0); ++i) {
    const int dim = input_indices_t(i, split_dim);
    const int slice_index = GetSliceIndex(dim, split_size, residual);
    const int slice_dim = values_inserted_in_slice[slice_index]++;
    output_values_t[slice_index](slice_dim) = input_values_t(i);
    for (int j = 0; j < num_dim; ++j) {
      const int64 original_dim = input_indices_t(i, j);
      output_indices_t[slice_index](slice_dim, j) =
          (j == split_dim)
              ? GetDimensionInSlice(original_dim, split_size, residual)
              : original_dim;
    }
  }

  result->clear();
  result->reserve(num_split);
  for (int i = 0; i < num_split; ++i) {
    SparseTensor tensor;
    Status create_status =
        Create(output_indices[i], output_values[i], output_shapes[i], &tensor);
    if (!create_status.ok()) {
      return create_status;
    }
    result->push_back(std::move(tensor));
  }
  return Status::OK();
}


  static inline int GetSliceIndex(const int dim, const int split_size,
                                  const int residual) {
    DCHECK_GT(split_size, 0);
    DCHECK_GE(dim, 0);
    if (residual == 0) return dim / split_size;
    const int offset = residual * (split_size + 1);
    if (dim < offset) {
      return dim / (split_size + 1);
    } else {
      return residual + ((dim - offset) / split_size);
    }
  }


  static inline int GetSliceShape(const int slice_index, const int split_size,
                                  const int residual) {
    DCHECK_GT(split_size, 0);
    DCHECK_GE(slice_index, 0);
    if (residual == 0) return split_size;
    if (slice_index < residual) {
      return split_size + 1;
    } else {
      return split_size;
    }
  }


  SparseTensor(Tensor ix, Tensor vals, const VarDimArray shape)
      : SparseTensor(std::move(ix), std::move(vals), shape,
                     UndefinedOrder(shape)) {}


zfile(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    char file_access[4];
    gs_parsed_file_name_t pname;
    int code = parse_file_access_string(op, file_access);
    stream *s;

    if (code < 0)
        return code;
    code = parse_file_name(op-1, &pname, i_ctx_p->LockFilePermissions, imemory);
    if (code < 0)
        return code;
        /*
         * HACK: temporarily patch the current context pointer into the
         * state pointer for stdio-related devices.  See ziodev.c for
         * more information.
         */
    if (pname.iodev && pname.iodev->dtype == iodev_dtype_stdio) {
        bool statement = (strcmp(pname.iodev->dname, "%statementedit%") == 0);
        bool lineedit = (strcmp(pname.iodev->dname, "%lineedit%") == 0);
        if (pname.fname)
            return_error(gs_error_invalidfileaccess);
        if (statement || lineedit) {
            /* These need special code to support callouts */
            gx_io_device *indev = gs_findiodevice(imemory,
                                                  (const byte *)"%stdin", 6);
            stream *ins;
            if (strcmp(file_access, "r"))
                return_error(gs_error_invalidfileaccess);
            indev->state = i_ctx_p;
            code = (indev->procs.open_device)(indev, file_access, &ins, imemory);
            indev->state = 0;
            if (code < 0)
                return code;
            check_ostack(2);
            push(2);
            make_stream_file(op - 3, ins, file_access);
            make_bool(op-2, statement);
            make_int(op-1, 0);
            make_string(op, icurrent_space, 0, NULL);
            return zfilelineedit(i_ctx_p);
        }
        pname.iodev->state = i_ctx_p;
        code = (*pname.iodev->procs.open_device)(pname.iodev,
                                                 file_access, &s, imemory);
        pname.iodev->state = NULL;
    } else {
        if (pname.iodev == NULL)
            pname.iodev = iodev_default(imemory);
        code = zopen_file(i_ctx_p, &pname, file_access, &s, imemory);
    }
    if (code < 0)
        return code;
    code = ssetfilename(s, op[-1].value.const_bytes, r_size(op - 1));
    if (code < 0) {
        sclose(s);
        return_error(gs_error_VMerror);
    }
    make_stream_file(op - 1, s, file_access);
    pop(1);
    return code;
}

Bug 694724: Have filenameforall and getenv honor SAFER


iodev_os_open_file(gx_io_device * iodev, const char *fname, uint len,
                   const char *file_access, stream ** ps, gs_memory_t * mem)
{
    return file_open_stream(fname, len, file_access,
                            file_default_buffer_size, ps,
                            iodev, iodev->procs.gp_fopen, mem);
}


file_continue(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    es_ptr pscratch = esp - 2;
    file_enum *pfen = r_ptr(esp - 1, file_enum);
    int devlen = esp[-3].value.intval;
    gx_io_device *iodev = r_ptr(esp - 4, gx_io_device);
    uint len = r_size(pscratch);
    uint code;

    if (len < devlen)
        return_error(gs_error_rangecheck);     /* not even room for device len */

    do {
        memcpy((char *)pscratch->value.bytes, iodev->dname, devlen);
        code = iodev->procs.enumerate_next(pfen, (char *)pscratch->value.bytes + devlen,
                    len - devlen);
        if (code == ~(uint) 0) {    /* all done */
            esp -= 5;               /* pop proc, pfen, devlen, iodev , mark */
            return o_pop_estack;
        } else if (code > len)      /* overran string */
            return_error(gs_error_rangecheck);
        else if (iodev != iodev_default(imemory)
              || (check_file_permissions_reduced(i_ctx_p, (char *)pscratch->value.bytes, code + devlen, "PermitFileReading")) == 0) {
            push(1);
            ref_assign(op, pscratch);
            r_set_size(op, code + devlen);
            push_op_estack(file_continue);  /* come again */
            *++esp = pscratch[2];   /* proc */
            return o_push_estack;
        }
    } while(1);
}


zfilenameforall(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    file_enum *pfen;
    gx_io_device *iodev = NULL;
    gs_parsed_file_name_t pname;
    int code = 0;

    check_write_type(*op, t_string);
    check_proc(op[-1]);
    check_read_type(op[-2], t_string);
    /* Push a mark, the iodev, devicenamelen, the scratch string, the enumerator, */
    /* and the procedure, and invoke the continuation. */
    check_estack(7);
    /* Get the iodevice */
    code = parse_file_name(op-2, &pname, i_ctx_p->LockFilePermissions, imemory);
    if (code < 0)
        return code;
    iodev = (pname.iodev == NULL) ? iodev_default(imemory) : pname.iodev;

    /* Check for several conditions that just cause us to return success */
    if (pname.len == 0 || iodev->procs.enumerate_files == iodev_no_enumerate_files) {
        pop(3);
        return 0;       /* no pattern, or device not found -- just return */
    }
    pfen = iodev->procs.enumerate_files(iodev, (const char *)pname.fname,
                pname.len, imemory);
    if (pfen == 0)
        return_error(gs_error_VMerror);
    push_mark_estack(es_for, file_cleanup);
    ++esp;
    make_istruct(esp, 0, iodev);
    ++esp;
    make_int(esp, r_size(op-2) - pname.len);
    *++esp = *op;
    ++esp;
    make_istruct(esp, 0, pfen);
    *++esp = op[-1];
    pop(3);
    code = file_continue(i_ctx_p);
    return (code == o_pop_estack ? o_push_estack : code);
}


execfile_finish(i_ctx_t *i_ctx_p)
{
    check_ostack(1);
    esp -= 2;
    execfile_cleanup(i_ctx_p);
    return o_pop_estack;
}


zopen_file(i_ctx_t *i_ctx_p, const gs_parsed_file_name_t *pfn,
           const char *file_access, stream **ps, gs_memory_t *mem)
{
    gx_io_device *const iodev = pfn->iodev;
    int code = 0;

    if (pfn->fname == NULL) {     /* just a device */
        iodev->state = i_ctx_p;
        code = iodev->procs.open_device(iodev, file_access, ps, mem);
        iodev->state = NULL;
        return code;
    }
    else {                      /* file */
        iodev_proc_open_file((*open_file)) = iodev->procs.open_file;

        if (open_file == 0)
            open_file = iodev_os_open_file;
        /* Check OS files to make sure we allow the type of access */
        if (open_file == iodev_os_open_file) {
            code = check_file_permissions(i_ctx_p, pfn->fname, pfn->len,
                file_access[0] == 'r' ? "PermitFileReading" : "PermitFileWriting");

            if (code < 0 && !file_is_tempfile(i_ctx_p,
                                          (const uchar *)pfn->fname, pfn->len))
                return code;
        }
        return open_file(iodev, pfn->fname, pfn->len, file_access, ps, mem);
    }
}


file_close(ref * pfile)
{
    stream *s;

    if (file_is_valid(s, pfile)) {      /* closing a closed file is a no-op */
        if (sclose(s))
            return_error(gs_error_ioerror);
    }
    return 0;
}


parse_file_access_string(const ref *op, char file_access[4])
{
    const byte *astr;

    check_read_type(*op, t_string);
    astr = op->value.const_bytes;
    switch (r_size(op)) {
        case 2:
            if (astr[1] != '+')
                return_error(gs_error_invalidfileaccess);
            file_access[1] = '+';
            file_access[2] = 0;
            break;
        case 1:
            file_access[1] = 0;
            break;
        default:
            return_error(gs_error_invalidfileaccess);
    }
    switch (astr[0]) {
        case 'r':
        case 'w':
        case 'a':
            break;
        default:
            return_error(gs_error_invalidfileaccess);
    }
    file_access[0] = astr[0];
    return 0;
}


parse_file_name(const ref * op, gs_parsed_file_name_t * pfn, bool safemode,
                gs_memory_t *memory)
{
    int code;

    check_read_type(*op, t_string);
    code = gs_parse_file_name(pfn, (const char *)op->value.const_bytes,
                              r_size(op), memory);
    if (code < 0)
        return code;
    /*
     * Check here for the %pipe device which is illegal when
     * LockFilePermissions is true. In the future we might want to allow
     * the %pipe device to be included on the PermitFile... paths, but
     * for now it is simply disallowed.
     */
    if (pfn->iodev && safemode && strcmp(pfn->iodev->dname, "%pipe%") == 0)
        return gs_error_invalidfileaccess;
    return code;
}


zfilenamesplit(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;

    check_read_type(*op, t_string);
/****** NOT IMPLEMENTED YET ******/
    return_error(gs_error_undefined);
}


ztempfile(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    const char *pstr;
    char fmode[4];
    int code = parse_file_access_string(op, fmode);
    char *prefix = NULL;
    char *fname= NULL;
    uint fnlen;
    FILE *sfile;
    stream *s;
    byte *buf, *sbody;

    if (code < 0)
        return code;
    prefix = (char *)gs_alloc_bytes(imemory, gp_file_name_sizeof, "ztempfile(prefix)");
    fname = (char *)gs_alloc_bytes(imemory, gp_file_name_sizeof, "ztempfile(fname)");
    if (!prefix || !fname) {
        code = gs_note_error(gs_error_VMerror);
        goto done;
    }

    strcat(fmode, gp_fmode_binary_suffix);
    if (r_has_type(op - 1, t_null))
        pstr = gp_scratch_file_name_prefix;
    else {
        uint psize;

        check_read_type(op[-1], t_string);
        psize = r_size(op - 1);
        if (psize >= gp_file_name_sizeof) {
            code = gs_note_error(gs_error_rangecheck);
            goto done;
        }
        memcpy(prefix, op[-1].value.const_bytes, psize);
        prefix[psize] = 0;
        pstr = prefix;
    }

    if (gp_file_name_is_absolute(pstr, strlen(pstr))) {
        if (check_file_permissions(i_ctx_p, pstr, strlen(pstr),
                                   "PermitFileWriting") < 0) {
            code = gs_note_error(gs_error_invalidfileaccess);
            goto done;
        }
    } else if (!prefix_is_simple(pstr)) {
        code = gs_note_error(gs_error_invalidfileaccess);
        goto done;
    }

    s = file_alloc_stream(imemory, "ztempfile(stream)");
    if (s == 0) {
        code = gs_note_error(gs_error_VMerror);
        goto done;
    }
    buf = gs_alloc_bytes(imemory, file_default_buffer_size,
                         "ztempfile(buffer)");
    if (buf == 0) {
        code = gs_note_error(gs_error_VMerror);
        goto done;
    }
    sfile = gp_open_scratch_file(imemory, pstr, fname, fmode);
    if (sfile == 0) {
        gs_free_object(imemory, buf, "ztempfile(buffer)");
        code = gs_note_error(gs_error_invalidfileaccess);
        goto done;
    }
    fnlen = strlen(fname);
    sbody = ialloc_string(fnlen, ".tempfile(fname)");
    if (sbody == 0) {
        gs_free_object(imemory, buf, "ztempfile(buffer)");
        code = gs_note_error(gs_error_VMerror);
        goto done;
    }
    memcpy(sbody, fname, fnlen);
    file_init_stream(s, sfile, fmode, buf, file_default_buffer_size);
    code = ssetfilename(s, (const unsigned char*) fname, fnlen);
    if (code < 0) {
        gx_io_device *iodev_dflt = iodev_default(imemory);
        sclose(s);
        iodev_dflt->procs.delete_file(iodev_dflt, fname);
        ifree_string(sbody, fnlen, ".tempfile(fname)");
        code = gs_note_error(gs_error_VMerror);
        goto done;
    }
    make_string(op - 1, a_readonly | icurrent_space, fnlen, sbody);
    make_stream_file(op, s, fmode);

done:
    if (prefix)
        gs_free_object(imemory, prefix, "ztempfile(prefix)");
    if (fname)
        gs_free_object(imemory, fname, "ztempfile(fname)");
    return code;
}


file_cleanup(i_ctx_t *i_ctx_p)
{
    gx_io_device *iodev = r_ptr(esp + 2, gx_io_device);

    iodev->procs.enumerate_close(r_ptr(esp + 5, file_enum));
    return 0;
}


zfilenamelistseparator(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;

    push(1);
    make_const_string(op, avm_foreign | a_readonly, 1,
                      (const byte *)&gp_file_name_list_separator);
    return 0;
}


lib_file_open_search_with_no_combine(gs_file_path_ptr  lib_path, const gs_memory_t *mem, i_ctx_t *i_ctx_p,
                                     const char *fname, uint flen, char *buffer, int blen, uint *pclen, ref *pfile,
                                     gx_io_device *iodev, bool starting_arg_file, char *fmode)
{
    stream *s;
    uint blen1 = blen;
    struct stat fstat;

    if (gp_file_name_reduce(fname, flen, buffer, &blen1) != gp_combine_success)
      goto skip;

    if (starting_arg_file || check_file_permissions_aux(i_ctx_p, buffer, blen1) >= 0) {
        if (iodev_os_open_file(iodev, (const char *)buffer, blen1,
                       (const char *)fmode, &s, (gs_memory_t *)mem) == 0) {
            *pclen = blen1;
            make_stream_file(pfile, s, "r");
            return 0;
        }
    }
    else {
        /* If we are not allowed to open the file by check_file_permissions_aux()
         * and if the file exists, throw an error.......
         * Otherwise, keep searching.
         */
        if ((*iodev->procs.file_status)(iodev,  buffer, &fstat) >= 0) {
            return_error(gs_error_invalidfileaccess);
        }
    }

 skip:
    return 1;
}


check_file_permissions_reduced(i_ctx_t *i_ctx_p, const char *fname, int len,
                        const char *permitgroup)
{
    long i;
    ref *permitlist = NULL;
    /* an empty string (first character == 0) if '\' character is */
    /* recognized as a file name separator as on DOS & Windows    */
    const char *win_sep2 = "\\";
    bool use_windows_pathsep = (gs_file_name_check_separator(win_sep2, 1, win_sep2) == 1);
    uint plen = gp_file_name_parents(fname, len);

    /* Assuming a reduced file name. */

    if (dict_find_string(&(i_ctx_p->userparams), permitgroup, &permitlist) <= 0)
        return 0;       /* if Permissions not found, just allow access */

    for (i=0; i<r_size(permitlist); i++) {
        ref permitstring;
        const string_match_params win_filename_params = {
                '*', '?', '\\', true, true      /* ignore case & '/' == '\\' */
        };
        const byte *permstr;
        uint permlen;
        int cwd_len = 0;

        if (array_get(imemory, permitlist, i, &permitstring) < 0 ||
            r_type(&permitstring) != t_string
           )
            break;      /* any problem, just fail */
        permstr = permitstring.value.bytes;
        permlen = r_size(&permitstring);
        /*
         * Check if any file name is permitted with "*".
         */
        if (permlen == 1 && permstr[0] == '*')
            return 0;           /* success */
        /*
         * If the filename starts with parent references,
         * the permission element must start with same number of parent references.
         */
        if (plen != 0 && plen != gp_file_name_parents((const char *)permstr, permlen))
            continue;
        cwd_len = gp_file_name_cwds((const char *)permstr, permlen);
        /*
         * If the permission starts with "./", absolute paths
         * are not permitted.
         */
        if (cwd_len > 0 && gp_file_name_is_absolute(fname, len))
            continue;
        /*
         * If the permission starts with "./", relative paths
         * with no "./" are allowed as well as with "./".
         * 'fname' has no "./" because it is reduced.
         */
        if (string_match( (const unsigned char*) fname, len,
                          permstr + cwd_len, permlen - cwd_len,
                use_windows_pathsep ? &win_filename_params : NULL))
            return 0;           /* success */
    }
    /* not found */
    return gs_error_invalidfileaccess;
}


zexecfile(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;

    check_type_access(*op, t_file, a_executable | a_read | a_execute);
    check_estack(4);            /* cleanup, file, finish, file */
    push_mark_estack(es_other, execfile_cleanup);
    *++esp = *op;
    push_op_estack(execfile_finish);
    return zexec(i_ctx_p);
}


lib_fopen(const gs_file_path_ptr pfpath, const gs_memory_t *mem, const char *fname)
{
    /* We need a buffer to hold the expanded file name. */
    char filename_found[DEFAULT_BUFFER_SIZE];
    FILE *file = NULL;
    uint fnamelen;
    ref obj;
    int code;

    /* open the usual 'stream', then if successful, return the file */
    code = lib_file_open(pfpath, mem, NULL, fname, strlen(fname),
                            filename_found, sizeof(filename_found), &fnamelen, &obj);

    if (code < 0)
        return NULL;
    file = ((stream *)(obj.value.pfile))->file;
    return file;
}


make_stream_file(ref * pfile, stream * s, const char *access)
{
    uint attrs =
        (access[1] == '+' ? a_write + a_read + a_execute : 0) |
        imemory_space((gs_ref_memory_t *) s->memory);

    if (access[0] == 'r') {
        make_file(pfile, attrs | (a_read | a_execute), s->read_id, s);
        s->write_id = 0;
    } else {
        make_file(pfile, attrs | a_write, s->write_id, s);
        s->read_id = 0;
    }
}


file_is_tempfile(i_ctx_t *i_ctx_p, const uchar *fname, int len)
{
    ref *SAFETY;
    ref *tempfiles;
    ref kname;

    if (dict_find_string(systemdict, "SAFETY", &SAFETY) <= 0 ||
            dict_find_string(SAFETY, "tempfiles", &tempfiles) <= 0)
        return false;
    if (name_ref(imemory, fname, len, &kname, -1) < 0 ||
            dict_find(tempfiles, &kname, &SAFETY) <= 0)
        return false;
    return true;
}


zrenamefile(i_ctx_t *i_ctx_p)
{
    int code;
    os_ptr op = osp;
    gs_parsed_file_name_t pname1, pname2;

    code = parse_real_file_name(op, &pname2, imemory, "renamefile(to)");
    if (code < 0)
        return code;

    pname1.fname = 0;
    code = parse_real_file_name(op - 1, &pname1, imemory, "renamefile(from)");
    if (code >= 0) {
        gx_io_device *iodev_dflt = iodev_default(imemory);
        if (pname1.iodev != pname2.iodev ) {
            if (pname1.iodev == iodev_dflt)
                pname1.iodev = pname2.iodev;
            if (pname2.iodev == iodev_dflt)
                pname2.iodev = pname1.iodev;
        }
        if (pname1.iodev != pname2.iodev ||
            (pname1.iodev == iodev_dflt &&
                /*
                 * We require FileControl permissions on the source path
                 * unless it is a temporary file. Also, we require FileControl
                 * and FileWriting permissions to the destination file/path.
                 */
              ((check_file_permissions(i_ctx_p, pname1.fname, pname1.len,
                                        "PermitFileControl") < 0 &&
                  !file_is_tempfile(i_ctx_p, op[-1].value.bytes, r_size(op - 1))) ||
              (check_file_permissions(i_ctx_p, pname2.fname, pname2.len,
                                        "PermitFileControl") < 0 ||
              check_file_permissions(i_ctx_p, pname2.fname, pname2.len,
                                        "PermitFileWriting") < 0 )))) {
            code = gs_note_error(gs_error_invalidfileaccess);
        } else {
            code = (*pname1.iodev->procs.rename_file)(pname1.iodev,
                            pname1.fname, pname2.fname);
        }
    }
    gs_free_file_name(&pname2, "renamefile(to)");
    gs_free_file_name(&pname1, "renamefile(from)");
    if (code < 0)
        return code;
    pop(2);
    return 0;
}


execfile_cleanup(i_ctx_t *i_ctx_p)
{
    check_ostack(1);
    *++osp = esp[2];
    return zclosefile(i_ctx_p);
}


zlibfile(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    int code;
    byte cname[DEFAULT_BUFFER_SIZE];
    uint clen;
    gs_parsed_file_name_t pname;
    stream *s;
    gx_io_device *iodev_dflt;

    check_ostack(2);
    code = parse_file_name(op, &pname, i_ctx_p->LockFilePermissions, imemory);
    if (code < 0)
        return code;
    iodev_dflt = iodev_default(imemory);
    if (pname.iodev == NULL)
        pname.iodev = iodev_dflt;
    if (pname.iodev != iodev_dflt) { /* Non-OS devices don't have search paths (yet). */
        code = zopen_file(i_ctx_p, &pname, "r", &s, imemory);
        if (code >= 0) {
            code = ssetfilename(s, op->value.const_bytes, r_size(op));
            if (code < 0) {
                sclose(s);
                return_error(gs_error_VMerror);
            }
        }
        if (code < 0) {
            push(1);
            make_false(op);
            return 0;
        }
        make_stream_file(op, s, "r");
    } else {
        ref fref;

        code = lib_file_open(i_ctx_p->lib_path, imemory, i_ctx_p, pname.fname, pname.len,
                             (char *)cname, sizeof(cname), &clen, &fref);
        if (code >= 0) {
            s = fptr(&fref);
            code = ssetfilename(s, cname, clen);
            if (code < 0) {
                sclose(s);
                return_error(gs_error_VMerror);
            }
        }
        if (code < 0) {
            if (code == gs_error_VMerror || code == gs_error_invalidfileaccess)
                return code;
            push(1);
            make_false(op);
            return 0;
        }
        ref_assign(op, &fref);
    }
    push(1);
    make_true(op);
    return 0;
}


parse_real_file_name(const ref *op, gs_parsed_file_name_t *pfn,
                     gs_memory_t *mem, client_name_t cname)
{
    check_read_type(*op, t_string);
    return gs_parse_real_file_name(pfn, (const char *)op->value.const_bytes,
                                   r_size(op), mem, cname);
}


check_file_permissions_aux(i_ctx_t *i_ctx_p, char *fname, uint flen)
{   /* i_ctx_p is NULL running init files. */
    /* fname must be reduced. */
    if (i_ctx_p == NULL)
        return 0;
    if (check_file_permissions_reduced(i_ctx_p, fname, flen, "PermitFileReading") < 0)
        return_error(gs_error_invalidfileaccess);
    return 0;
}


lib_file_open(gs_file_path_ptr  lib_path, const gs_memory_t *mem, i_ctx_t *i_ctx_p,
                       const char *fname, uint flen, char *buffer, int blen, uint *pclen, ref *pfile)
{   /* i_ctx_p is NULL running arg (@) files.
     * lib_path and mem are never NULL
     */
    bool starting_arg_file = (i_ctx_p == NULL) ? true : i_ctx_p->starting_arg_file;
    bool search_with_no_combine = false;
    bool search_with_combine = false;
    char fmode[2] = { 'r', 0};
    gx_io_device *iodev = iodev_default(mem);
    gs_main_instance *minst = get_minst_from_memory(mem);
    int code;

    /* when starting arg files (@ files) iodev_default is not yet set */
    if (iodev == 0)
        iodev = (gx_io_device *)gx_io_device_table[0];

    if (gp_file_name_is_absolute(fname, flen)) {
       search_with_no_combine = true;
       search_with_combine = false;
    } else {
       search_with_no_combine = starting_arg_file;
       search_with_combine = true;
    }
    if (minst->search_here_first) {
      if (search_with_no_combine) {
        code = lib_file_open_search_with_no_combine(lib_path, mem, i_ctx_p,
                                                    fname, flen, buffer, blen, pclen, pfile,
                                                    iodev, starting_arg_file, fmode);
        if (code <= 0) /* +ve means continue continue */
          return code;
      }
      if (search_with_combine) {
        code = lib_file_open_search_with_combine(lib_path, mem, i_ctx_p,
                                                 fname, flen, buffer, blen, pclen, pfile,
                                                 iodev, starting_arg_file, fmode);
        if (code <= 0) /* +ve means continue searching */
          return code;
      }
    } else {
      if (search_with_combine) {
        code = lib_file_open_search_with_combine(lib_path, mem, i_ctx_p,
                                                 fname, flen, buffer, blen, pclen, pfile,
                                                 iodev, starting_arg_file, fmode);
        if (code <= 0) /* +ve means continue searching */
          return code;
      }
      if (search_with_no_combine) {
        code = lib_file_open_search_with_no_combine(lib_path, mem, i_ctx_p,
                                                    fname, flen, buffer, blen, pclen, pfile,
                                                    iodev, starting_arg_file, fmode);
        if (code <= 0) /* +ve means continue searching */
          return code;
      }
    }
    return_error(gs_error_undefinedfilename);
}


filter_report_error(stream_state * st, const char *str)
{
    if_debug1m('s', st->memory, "[s]stream error: %s\n", str);
    strncpy(st->error_string, str, STREAM_MAX_ERROR_STRING);
    /* Ensure null termination. */
    st->error_string[STREAM_MAX_ERROR_STRING] = 0;
    return 0;
}


filter_open(const char *file_access, uint buffer_size, ref * pfile,
            const stream_procs * procs, const stream_template * templat,
            const stream_state * st, gs_memory_t *mem)
{
    stream *s;
    uint ssize = gs_struct_type_size(templat->stype);
    stream_state *sst = 0;
    int code;

    if (templat->stype != &st_stream_state) {
        sst = s_alloc_state(mem, templat->stype, "filter_open(stream_state)");
        if (sst == 0)
            return_error(gs_error_VMerror);
    }
    code = file_open_stream((char *)0, 0, file_access, buffer_size, &s,
                                (gx_io_device *)0, (iodev_proc_fopen_t)0, mem);
    if (code < 0) {
        gs_free_object(mem, sst, "filter_open(stream_state)");
        return code;
    }
    s_std_init(s, s->cbuf, s->bsize, procs,
               (*file_access == 'r' ? s_mode_read : s_mode_write));
    s->procs.process = templat->process;
    s->save_close = s->procs.close;
    s->procs.close = file_close_file;
    if (sst == 0) {
        /* This stream doesn't have any state of its own. */
        /* Hack: use the stream itself as the state. */
        sst = (stream_state *) s;
    } else if (st != 0)         /* might not have client parameters */
        memcpy(sst, st, ssize);
    s->state = sst;
    s_init_state(sst, templat, mem);
    sst->report_error = filter_report_error;

    if (templat->init != 0) {
        code = (*templat->init)(sst);
        if (code < 0) {
            gs_free_object(mem, sst, "filter_open(stream_state)");
            gs_free_object(mem, s->cbuf, "filter_open(buffer)");
            return code;
        }
    }
    make_stream_file(pfile, s, file_access);
    return 0;
}


zdeletefile(i_ctx_t *i_ctx_p)
{
    os_ptr op = osp;
    gs_parsed_file_name_t pname;
    int code = parse_real_file_name(op, &pname, imemory, "deletefile");

    if (code < 0)
        return code;
    if (pname.iodev == iodev_default(imemory)) {
        if ((code = check_file_permissions(i_ctx_p, pname.fname, pname.len,
                "PermitFileControl")) < 0 &&
                 !file_is_tempfile(i_ctx_p, op->value.bytes, r_size(op))) {
            return code;
        }
    }
    code = (*pname.iodev->procs.delete_file)(pname.iodev, pname.fname);
    gs_free_file_name(&pname, "deletefile");
    if (code < 0)
        return code;
    pop(1);
    return 0;
}


file_read_string(const byte *str, uint len, ref *pfile, gs_ref_memory_t *imem)
{
    stream *s = file_alloc_stream((gs_memory_t *)imem, "file_read_string");

    if (s == 0)
        return_error(gs_error_VMerror);
    sread_string(s, str, len);
    s->foreign = 1;
    s->write_id = 0;
    make_file(pfile, a_readonly | imemory_space(imem), s->read_id, s);
    s->save_close = s->procs.close;
    s->procs.close = file_close_disable;
    return 0;
}


check_file_permissions(i_ctx_t *i_ctx_p, const char *fname, int len,
                        const char *permitgroup)
{
    char fname_reduced[gp_file_name_sizeof];
    uint rlen = sizeof(fname_reduced);

    if (gp_file_name_reduce(fname, len, fname_reduced, &rlen) != gp_combine_success)
        return gs_error_invalidaccess;         /* fail if we couldn't reduce */
    return check_file_permissions_reduced(i_ctx_p, fname_reduced, rlen, permitgroup);
}


make_invalid_file(i_ctx_t *i_ctx_p, ref * fp)
{
    make_file(fp, avm_invalid_file_entry, ~0, i_ctx_p->invalid_file_stream);
}


lib_file_open_search_with_combine(gs_file_path_ptr  lib_path, const gs_memory_t *mem, i_ctx_t *i_ctx_p,
                                  const char *fname, uint flen, char *buffer, int blen, uint *pclen, ref *pfile,
                                  gx_io_device *iodev, bool starting_arg_file, char *fmode)
{
    stream *s;
    const gs_file_path *pfpath = lib_path;
    uint pi;

    for (pi = 0; pi < r_size(&pfpath->list); ++pi) {
        const ref *prdir = pfpath->list.value.refs + pi;
        const char *pstr = (const char *)prdir->value.const_bytes;
        uint plen = r_size(prdir), blen1 = blen;
        gs_parsed_file_name_t pname;
        gp_file_name_combine_result r;

        /* We need to concatenate and parse the file name here
         * if this path has a %device% prefix.              */
        if (pstr[0] == '%') {
            int code;

            /* We concatenate directly since gp_file_name_combine_*
             * rules are not correct for other devices such as %rom% */
            code = gs_parse_file_name(&pname, pstr, plen, mem);
            if (code < 0)
                continue;
            if (blen < max(pname.len, plen) + flen)
            	return_error(gs_error_limitcheck);
            memcpy(buffer, pname.fname, pname.len);
            memcpy(buffer+pname.len, fname, flen);
            code = pname.iodev->procs.open_file(pname.iodev, buffer, pname.len + flen, fmode,
                                          &s, (gs_memory_t *)mem);
            if (code < 0)
                continue;
            make_stream_file(pfile, s, "r");
            /* fill in the buffer with the device concatenated */
            memcpy(buffer, pstr, plen);
            memcpy(buffer+plen, fname, flen);
            *pclen = plen + flen;
            return 0;
        } else {
            r = gp_file_name_combine(pstr, plen,
                    fname, flen, false, buffer, &blen1);
            if (r != gp_combine_success)
                continue;
            if (iodev_os_open_file(iodev, (const char *)buffer, blen1, (const char *)fmode,
                                    &s, (gs_memory_t *)mem) == 0) {
                if (starting_arg_file ||
                    check_file_permissions_aux(i_ctx_p, buffer, blen1) >= 0) {
                    *pclen = blen1;
                    make_stream_file(pfile, s, "r");
                    return 0;
                }
                sclose(s);
                return_error(gs_error_invalidfileaccess);
            }
        }
    }
    return 1;
}


calculateNumYLevels (const TileDescription& tileDesc,
		     int minX, int maxX,
		     int minY, int maxY)
{
    int num = 0;

    switch (tileDesc.mode)
    {
      case ONE_LEVEL:

	num = 1;
	break;

      case MIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  int h = maxY - minY + 1;
	  num = roundLog2 (std::max (w, h), tileDesc.roundingMode) + 1;
	}
        break;

      case RIPMAP_LEVELS:

	{
	  int h = maxY - minY + 1;
	  num = roundLog2 (h, tileDesc.roundingMode) + 1;
	}
	break;

      default:

	throw IEX_NAMESPACE::ArgExc ("Unknown LevelMode format.");
    }

    return num;
}

Avoid overflow in calculateNumTiles when size=MAX_INT (#825)

* Avoid overflow in calculateNumTiles when size=MAX_INT

Signed-off-by: Cary Phillips <cary@ilm.com>

* Compute level size with 64 bits to avoid overflow

Signed-off-by: Cary Phillips <cary@ilm.com>

OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER

using IMATH_NAMESPACE::Box2i;


precalculateTileInfo (const TileDescription& tileDesc,
		      int minX, int maxX,
		      int minY, int maxY,
		      int *&numXTiles, int *&numYTiles,
		      int &numXLevels, int &numYLevels)
{
    numXLevels = calculateNumXLevels(tileDesc, minX, maxX, minY, maxY);
    numYLevels = calculateNumYLevels(tileDesc, minX, maxX, minY, maxY);

    numXTiles = new int[numXLevels];
    numYTiles = new int[numYLevels];

    calculateNumTiles (numXTiles,
		       numXLevels,
		       minX, maxX,
		       tileDesc.xSize,
		       tileDesc.roundingMode);

    calculateNumTiles (numYTiles,
		       numYLevels,
		       minY, maxY,
		       tileDesc.ySize,
		       tileDesc.roundingMode);
}


calculateBytesPerPixel (const Header &header)
{
    const ChannelList &channels = header.channels();

    size_t bytesPerPixel = 0;

    for (ChannelList::ConstIterator c = channels.begin();
	 c != channels.end();
	 ++c)
    {
	bytesPerPixel += pixelTypeSize (c.channel().type);
    }

    return bytesPerPixel;
}


getTiledChunkOffsetTableSize(const Header& header)
{
    //
    // Save the dataWindow information
    //

    const Box2i &dataWindow = header.dataWindow();

    //
    // Precompute level and tile information.
    //

    int* numXTiles=nullptr;
    int* numYTiles=nullptr;
    int numXLevels;
    int numYLevels;
    try
    {
        precalculateTileInfo (header.tileDescription(),
                            dataWindow.min.x, dataWindow.max.x,
                            dataWindow.min.y, dataWindow.max.y,
                            numXTiles, numYTiles,
                            numXLevels, numYLevels);

        //
        // Calculate lineOffsetSize.
        //
        Int64 lineOffsetSize = 0;
        const TileDescription &desc = header.tileDescription();
        switch (desc.mode)
        {
            case ONE_LEVEL:
            case MIPMAP_LEVELS:
                for (int i = 0; i < numXLevels; i++)
                {
                    lineOffsetSize += static_cast<Int64>(numXTiles[i]) * static_cast<Int64>(numYTiles[i]);
                    if ( lineOffsetSize > static_cast<Int64>(std::numeric_limits<int>::max()) )
                    {
                        throw IEX_NAMESPACE::LogicExc("Maximum number of tiles exceeded");
                    }
                }
            break;
            case RIPMAP_LEVELS:
                for (int i = 0; i < numXLevels; i++)
                {
                    for (int j = 0; j < numYLevels; j++)
                    {
                        lineOffsetSize += static_cast<Int64>(numXTiles[i]) * static_cast<Int64>(numYTiles[j]);
                        if ( lineOffsetSize > static_cast<Int64>(std::numeric_limits<int>::max()) )
                        {
                            throw IEX_NAMESPACE::LogicExc("Maximum number of tiles exceeded");
                        }
                    }
                }
            break;
            case NUM_LEVELMODES :
                throw IEX_NAMESPACE::LogicExc("Bad level mode getting chunk offset table size");
        }
        delete[] numXTiles;
        delete[] numYTiles;

        return static_cast<int>(lineOffsetSize);

    }
    catch(...)
    {
        delete[] numXTiles;
        delete[] numYTiles;

        throw;
    }

}


calculateBytesPerLine (const Header &header,
                       char* sampleCountBase,
                       int sampleCountXStride,
                       int sampleCountYStride,
                       int minX, int maxX,
                       int minY, int maxY,
                       std::vector<int>& xOffsets,
                       std::vector<int>& yOffsets,
                       std::vector<Int64>& bytesPerLine)
{
    const ChannelList &channels = header.channels();

    int pos = 0;
    for (ChannelList::ConstIterator c = channels.begin();
         c != channels.end();
         ++c, ++pos)
    {
        int xOffset = xOffsets[pos];
        int yOffset = yOffsets[pos];
        int i = 0;
        for (int y = minY - yOffset; y <= maxY - yOffset; y++, i++)
            for (int x = minX - xOffset; x <= maxX - xOffset; x++)
            {
                bytesPerLine[i] += sampleCount(sampleCountBase,
                                               sampleCountXStride,
                                               sampleCountYStride,
                                               x, y)
                                   * pixelTypeSize (c.channel().type);
            }
    }
}


dataWindowForTile (const TileDescription &tileDesc,
		   int minX, int maxX,
		   int minY, int maxY,
		   int dx, int dy,
		   int lx, int ly)
{
    V2i tileMin = V2i (minX + dx * tileDesc.xSize,
		       minY + dy * tileDesc.ySize);

    V2i tileMax = tileMin + V2i (tileDesc.xSize - 1, tileDesc.ySize - 1);

    V2i levelMax = dataWindowForLevel
		       (tileDesc, minX, maxX, minY, maxY, lx, ly).max;

    tileMax = V2i (std::min (tileMax[0], levelMax[0]),
		   std::min (tileMax[1], levelMax[1]));

    return Box2i (tileMin, tileMax);
}


floorLog2 (int x)
{
    //
    // For x > 0, floorLog2(y) returns floor(log(x)/log(2)).
    //

    int y = 0;

    while (x > 1)
    {
	y +=  1;
	x >>= 1;
    }

    return y;
}


levelSize (int min, int max, int l, LevelRoundingMode rmode)
{
    if (l < 0)
	throw IEX_NAMESPACE::ArgExc ("Argument not in valid range.");

    int a = max - min + 1;
    int b = (1 << l);
    int size = a / b;

    if (rmode == ROUND_UP && size * b < a)
	size += 1;

    return std::max (size, 1);
}


dataWindowForLevel (const TileDescription &tileDesc,
		    int minX, int maxX,
		    int minY, int maxY,
		    int lx, int ly)
{
    V2i levelMin = V2i (minX, minY);

    V2i levelMax = levelMin +
		   V2i (levelSize (minX, maxX, lx, tileDesc.roundingMode) - 1,
			levelSize (minY, maxY, ly, tileDesc.roundingMode) - 1);

    return Box2i(levelMin, levelMax);
}


calculateNumTiles (int *numTiles,
		   int numLevels,
		   int min, int max,
		   int size,
		   LevelRoundingMode rmode)
{
    for (int i = 0; i < numLevels; i++)
    {
        // use 64 bits to avoid int overflow if size is large.
        Int64 l = levelSize (min, max, i, rmode);
        numTiles[i] = (l + size - 1) / size;
    }
}


ceilLog2 (int x)
{
    //
    // For x > 0, ceilLog2(y) returns ceil(log(x)/log(2)).
    //

    int y = 0;
    int r = 0;

    while (x > 1)
    {
	if (x & 1)
	    r = 1;

	y +=  1;
	x >>= 1;
    }

    return y + r;
}


calculateNumXLevels (const TileDescription& tileDesc,
		     int minX, int maxX,
		     int minY, int maxY)
{
    int num = 0;

    switch (tileDesc.mode)
    {
      case ONE_LEVEL:

	num = 1;
	break;

      case MIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  int h = maxY - minY + 1;
	  num = roundLog2 (std::max (w, h), tileDesc.roundingMode) + 1;
	}
        break;

      case RIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  num = roundLog2 (w, tileDesc.roundingMode) + 1;
	}
	break;

      default:

	throw IEX_NAMESPACE::ArgExc ("Unknown LevelMode format.");
    }

    return num;
}


roundLog2 (int x, LevelRoundingMode rmode)
{
    return (rmode == ROUND_DOWN)? floorLog2 (x): ceilLog2 (x);
}


set_omnifunc_option(void)
{
    int	retval;

    retval = option_set_callback_func(curbuf->b_p_ofu, &ofu_cb);
    if (retval == OK)
	set_buflocal_ofu_callback(curbuf);

    return retval;
}

patch 9.0.0102: reading past end of line with insert mode completion

Problem:    Reading past end of line with insert mode completion.
Solution:   Check text length.

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

static int	  compl_enter_selects = FALSE;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static void spell_back_to_badword(void);

static int  spell_bad_len = 0;

static int compl_match_arraysize;

static callback_T ofu_cb;

typedef struct
{
    char_u	*e_cpt;			// current entry in 'complete'
    buf_T	*ins_buf;		// buffer being scanned
    pos_T	*cur_match_pos;			// current match position
    pos_T	prev_match_pos;		// previous match position
    int		set_match_pos;		// save first_match_pos/last_match_pos
    pos_T	first_match_pos;	// first match position
    pos_T	last_match_pos;		// last match position
    int		found_all;		// found all matches of a certain type.
    char_u	*dict;			// dictionary file to search
    int		dict_f;			// "dict" is an exact file name or not
} ins_compl_next_state_T;


pum_wanted(void)
{
    // 'completeopt' must contain "menu" or "menuone"
    if (vim_strchr(p_cot, 'm') == NULL)
	return FALSE;

    // The display looks bad on a B&W display.
    if (t_colors < 8
#ifdef FEAT_GUI
	    && !gui.in_use
#endif
	    )
	return FALSE;
    return TRUE;
}

static char *ctrl_x_msgs[] =
{
    N_(" Keyword completion (^N^P)"), // CTRL_X_NORMAL, ^P/^N compl.
    N_(" ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)"),
    NULL, // CTRL_X_SCROLL: depends on state
    N_(" Whole line completion (^L^N^P)"),
    N_(" File name completion (^F^N^P)"),
    N_(" Tag completion (^]^N^P)"),
    N_(" Path pattern completion (^N^P)"),
    N_(" Definition completion (^D^N^P)"),
    NULL, // CTRL_X_FINISHED
    N_(" Dictionary completion (^K^N^P)"),
    N_(" Thesaurus completion (^T^N^P)"),
    N_(" Command-line completion (^V^N^P)"),
    N_(" User defined completion (^U^N^P)"),
    N_(" Omni completion (^O^N^P)"),
    N_(" Spelling suggestion (s^N^P)"),
    N_(" Keyword Local completion (^N^P)"),
    NULL,   // CTRL_X_EVAL doesn't use msg.
    N_(" Command-line completion (^V^N^P)"),
};

static char *ctrl_x_mode_names[] = {
	"keyword",
	"ctrl_x",
	"scroll",
	"whole_line",
	"files",
	"tags",
	"path_patterns",
	"path_defines",
	"unknown",	    // CTRL_X_FINISHED
	"dictionary",
	"thesaurus",
	"cmdline",
	"function",
	"omni",
	"spell",
	NULL,		    // CTRL_X_LOCAL_MSG only used in "ctrl_x_msgs"
	"eval",
	"cmdline",
};

#define CPT_MENU	1	// "menu"

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

# define CP_EQUAL	    8	// ins_compl_equal() always returns TRUE

static compl_T    *compl_first_match = NULL;

static compl_T    *compl_curr_match = NULL;

static compl_T    *compl_shown_match = NULL;

static compl_T    *compl_old_match = NULL;

static int	  compl_enter_selects = FALSE;

static char_u	  *compl_leader = NULL;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static char_u	  *compl_pattern = NULL;

static char_u	  *compl_orig_text = NULL;

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);


get_insert_callback(int type)
{
    if (type == CTRL_X_FUNCTION)
	return &curbuf->b_cfu_cb;
    if (type == CTRL_X_OMNI)
	return &curbuf->b_ofu_cb;
    // CTRL_X_THESAURUS
    return (*curbuf->b_p_tsrfu != NUL) ? &curbuf->b_tsrfu_cb : &tsrfu_cb;
}

# define CTRL_X_THESAURUS	(10 + CTRL_X_WANT_IDENT)

# define CTRL_X_FUNCTION	12

# define CTRL_X_OMNI		13

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

static int	  compl_enter_selects = FALSE;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static int  ins_compl_need_restart(void);

static int  get_compl_len(void);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static int  spell_bad_len = 0;

static int compl_match_arraysize;

static callback_T tsrfu_cb;

typedef struct
{
    char_u	*e_cpt;			// current entry in 'complete'
    buf_T	*ins_buf;		// buffer being scanned
    pos_T	*cur_match_pos;			// current match position
    pos_T	prev_match_pos;		// previous match position
    int		set_match_pos;		// save first_match_pos/last_match_pos
    pos_T	first_match_pos;	// first match position
    pos_T	last_match_pos;		// last match position
    int		found_all;		// found all matches of a certain type.
    char_u	*dict;			// dictionary file to search
    int		dict_f;			// "dict" is an exact file name or not
} ins_compl_next_state_T;


int ctrl_x_mode_spell(void) { return ctrl_x_mode == CTRL_X_SPELL; }

# define CTRL_X_SPELL		14

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

static int	  compl_enter_selects = FALSE;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

# define CONT_LOCAL	32	// for ctrl_x_mode 0, ^X^P/^X^N do a local

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static void spell_back_to_badword(void);

static int  spell_bad_len = 0;

static int compl_match_arraysize;


ctrl_x_mode_not_default(void)
{
    return ctrl_x_mode != CTRL_X_NORMAL;
}

# define CTRL_X_NORMAL		0  // CTRL-N CTRL-P completion, default

static char *ctrl_x_msgs[] =
{
    N_(" Keyword completion (^N^P)"), // CTRL_X_NORMAL, ^P/^N compl.
    N_(" ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)"),
    NULL, // CTRL_X_SCROLL: depends on state
    N_(" Whole line completion (^L^N^P)"),
    N_(" File name completion (^F^N^P)"),
    N_(" Tag completion (^]^N^P)"),
    N_(" Path pattern completion (^N^P)"),
    N_(" Definition completion (^D^N^P)"),
    NULL, // CTRL_X_FINISHED
    N_(" Dictionary completion (^K^N^P)"),
    N_(" Thesaurus completion (^T^N^P)"),
    N_(" Command-line completion (^V^N^P)"),
    N_(" User defined completion (^U^N^P)"),
    N_(" Omni completion (^O^N^P)"),
    N_(" Spelling suggestion (s^N^P)"),
    N_(" Keyword Local completion (^N^P)"),
    NULL,   // CTRL_X_EVAL doesn't use msg.
    N_(" Command-line completion (^V^N^P)"),
};

static int	  ctrl_x_mode = CTRL_X_NORMAL;

# define CONT_LOCAL	32	// for ctrl_x_mode 0, ^X^P/^X^N do a local

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static void spell_back_to_badword(void);


ins_compl_enter_selects(void)
{
    return compl_enter_selects;
}

static int	  compl_enter_selects = FALSE;

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static void spell_back_to_badword(void);


get_complete_funcname(int type)
{
    switch (type)
    {
	case CTRL_X_FUNCTION:
	    return curbuf->b_p_cfu;
	case CTRL_X_OMNI:
	    return curbuf->b_p_ofu;
	case CTRL_X_THESAURUS:
	    return *curbuf->b_p_tsrfu == NUL ? p_tsrfu : curbuf->b_p_tsrfu;
	default:
	    return (char_u *)"";
    }
}

# define CTRL_X_NORMAL		0  // CTRL-N CTRL-P completion, default

# define CTRL_X_THESAURUS	(10 + CTRL_X_WANT_IDENT)

# define CTRL_X_FUNCTION	12

# define CTRL_X_OMNI		13

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

# define CP_ICASE	    16	// ins_compl_equal() ignores case

static int	  compl_enter_selects = FALSE;

static char_u	  *compl_leader = NULL;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static char_u	  *compl_pattern = NULL;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static char_u	  *compl_orig_text = NULL;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static char_u *find_line_end(char_u *ptr);

static int  ins_compl_need_restart(void);

static int  get_compl_len(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static int  spell_bad_len = 0;

static int compl_match_arraysize;


show_pum(int prev_w_wrow, int prev_w_leftcol)
{
    // RedrawingDisabled may be set when invoked through complete().
    int n = RedrawingDisabled;

    RedrawingDisabled = 0;

    // If the cursor moved or the display scrolled we need to remove the pum
    // first.
    setcursor();
    if (prev_w_wrow != curwin->w_wrow || prev_w_leftcol != curwin->w_leftcol)
	ins_compl_del_pum();

    ins_compl_show_pum();
    setcursor();
    RedrawingDisabled = n;
}

# define CTRL_X_EVAL		16	// for builtin function complete()

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

# define CP_ORIGINAL_TEXT   1	// the original text when the expansion begun

static int	  compl_enter_selects = FALSE;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

# define CONT_INTRPT	(2 + 4)	// a ^X interrupted the current expansion

# define CONT_S_IPOS	8	// next ^X<> will set initial_pos?

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static int  ins_compl_need_restart(void);

static int  get_compl_len(void);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static int  spell_bad_len = 0;

static int compl_match_arraysize;

#define DICT_FIRST	(1)	// use just first element in "dict"

#define DICT_EXACT	(2)	// "dict" is the exact name of a file

typedef struct
{
    char_u	*e_cpt;			// current entry in 'complete'
    buf_T	*ins_buf;		// buffer being scanned
    pos_T	*cur_match_pos;			// current match position
    pos_T	prev_match_pos;		// previous match position
    int		set_match_pos;		// save first_match_pos/last_match_pos
    pos_T	first_match_pos;	// first match position
    pos_T	last_match_pos;		// last match position
    int		found_all;		// found all matches of a certain type.
    char_u	*dict;			// dictionary file to search
    int		dict_f;			// "dict" is an exact file name or not
} ins_compl_next_state_T;


compl_status_clear(void)
{
    compl_cont_status = 0;
}

# define CP_CONT_S_IPOS	    4	// use CONT_S_IPOS for compl_cont_status

static int	  compl_cont_status = 0;

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static void spell_back_to_badword(void);


get_complete_info(list_T *what_list, dict_T *retdict)
{
    int		ret = OK;
    listitem_T	*item;
#define CI_WHAT_MODE		0x01
#define CI_WHAT_PUM_VISIBLE	0x02
#define CI_WHAT_ITEMS		0x04
#define CI_WHAT_SELECTED	0x08
#define CI_WHAT_INSERTED	0x10
#define CI_WHAT_ALL		0xff
    int		what_flag;

    if (what_list == NULL)
	what_flag = CI_WHAT_ALL;
    else
    {
	what_flag = 0;
	CHECK_LIST_MATERIALIZE(what_list);
	FOR_ALL_LIST_ITEMS(what_list, item)
	{
	    char_u *what = tv_get_string(&item->li_tv);

	    if (STRCMP(what, "mode") == 0)
		what_flag |= CI_WHAT_MODE;
	    else if (STRCMP(what, "pum_visible") == 0)
		what_flag |= CI_WHAT_PUM_VISIBLE;
	    else if (STRCMP(what, "items") == 0)
		what_flag |= CI_WHAT_ITEMS;
	    else if (STRCMP(what, "selected") == 0)
		what_flag |= CI_WHAT_SELECTED;
	    else if (STRCMP(what, "inserted") == 0)
		what_flag |= CI_WHAT_INSERTED;
	}
    }

    if (ret == OK && (what_flag & CI_WHAT_MODE))
	ret = dict_add_string(retdict, "mode", ins_compl_mode());

    if (ret == OK && (what_flag & CI_WHAT_PUM_VISIBLE))
	ret = dict_add_number(retdict, "pum_visible", pum_visible());

    if (ret == OK && (what_flag & CI_WHAT_ITEMS))
    {
	list_T	    *li;
	dict_T	    *di;
	compl_T     *match;

	li = list_alloc();
	if (li == NULL)
	    return;
	ret = dict_add_list(retdict, "items", li);
	if (ret == OK && compl_first_match != NULL)
	{
	    match = compl_first_match;
	    do
	    {
		if (!match_at_original_text(match))
		{
		    di = dict_alloc();
		    if (di == NULL)
			return;
		    ret = list_append_dict(li, di);
		    if (ret != OK)
			return;
		    dict_add_string(di, "word", match->cp_str);
		    dict_add_string(di, "abbr", match->cp_text[CPT_ABBR]);
		    dict_add_string(di, "menu", match->cp_text[CPT_MENU]);
		    dict_add_string(di, "kind", match->cp_text[CPT_KIND]);
		    dict_add_string(di, "info", match->cp_text[CPT_INFO]);
		    if (match->cp_user_data.v_type == VAR_UNKNOWN)
			// Add an empty string for backwards compatibility
			dict_add_string(di, "user_data", (char_u *)"");
		    else
			dict_add_tv(di, "user_data", &match->cp_user_data);
		}
		match = match->cp_next;
	    }
	    while (match != NULL && !is_first_match(match));
	}
    }

    if (ret == OK && (what_flag & CI_WHAT_SELECTED))
    {
	if (compl_curr_match != NULL && compl_curr_match->cp_number == -1)
	    ins_compl_update_sequence_numbers();
	ret = dict_add_number(retdict, "selected", compl_curr_match != NULL
				      ? compl_curr_match->cp_number - 1 : -1);
    }

    // TODO
    // if (ret == OK && (what_flag & CI_WHAT_INSERTED))
}

# define CTRL_X_WANT_IDENT	0x100

# define CTRL_X_NORMAL		0  // CTRL-N CTRL-P completion, default

# define CTRL_X_NOT_DEFINED_YET	1

# define CTRL_X_SCROLL		2

# define CTRL_X_WHOLE_LINE	3

# define CTRL_X_FILES		4

# define CTRL_X_TAGS		(5 + CTRL_X_WANT_IDENT)

# define CTRL_X_PATH_PATTERNS	(6 + CTRL_X_WANT_IDENT)

# define CTRL_X_PATH_DEFINES	(7 + CTRL_X_WANT_IDENT)

# define CTRL_X_FINISHED		8

# define CTRL_X_DICTIONARY	(9 + CTRL_X_WANT_IDENT)

# define CTRL_X_THESAURUS	(10 + CTRL_X_WANT_IDENT)

# define CTRL_X_CMDLINE		11

# define CTRL_X_FUNCTION	12

# define CTRL_X_OMNI		13

# define CTRL_X_SPELL		14

# define CTRL_X_LOCAL_MSG	15	// only used in "ctrl_x_msgs"

# define CTRL_X_EVAL		16	// for builtin function complete()

# define CTRL_X_CMDLINE_CTRL_X	17	// CTRL-X typed in CTRL_X_CMDLINE

# define CTRL_X_MSG(i) ctrl_x_msgs[(i) & ~CTRL_X_WANT_IDENT]

static char *ctrl_x_msgs[] =
{
    N_(" Keyword completion (^N^P)"), // CTRL_X_NORMAL, ^P/^N compl.
    N_(" ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)"),
    NULL, // CTRL_X_SCROLL: depends on state
    N_(" Whole line completion (^L^N^P)"),
    N_(" File name completion (^F^N^P)"),
    N_(" Tag completion (^]^N^P)"),
    N_(" Path pattern completion (^N^P)"),
    N_(" Definition completion (^D^N^P)"),
    NULL, // CTRL_X_FINISHED
    N_(" Dictionary completion (^K^N^P)"),
    N_(" Thesaurus completion (^T^N^P)"),
    N_(" Command-line completion (^V^N^P)"),
    N_(" User defined completion (^U^N^P)"),
    N_(" Omni completion (^O^N^P)"),
    N_(" Spelling suggestion (s^N^P)"),
    N_(" Keyword Local completion (^N^P)"),
    NULL,   // CTRL_X_EVAL doesn't use msg.
    N_(" Command-line completion (^V^N^P)"),
};

static char *ctrl_x_mode_names[] = {
	"keyword",
	"ctrl_x",
	"scroll",
	"whole_line",
	"files",
	"tags",
	"path_patterns",
	"path_defines",
	"unknown",	    // CTRL_X_FINISHED
	"dictionary",
	"thesaurus",
	"cmdline",
	"function",
	"omni",
	"spell",
	NULL,		    // CTRL_X_LOCAL_MSG only used in "ctrl_x_msgs"
	"eval",
	"cmdline",
};

#define CPT_ABBR	0	// "abbr"

#define CPT_MENU	1	// "menu"

#define CPT_KIND	2	// "kind"

#define CPT_INFO	3	// "info"

#define CPT_COUNT	4	// Number of entries

typedef struct compl_S compl_T;


int ctrl_x_mode_path_defines(void) {
				   return ctrl_x_mode == CTRL_X_PATH_DEFINES; }

# define CTRL_X_PATH_DEFINES	(7 + CTRL_X_WANT_IDENT)

struct compl_S
{
    compl_T	*cp_next;
    compl_T	*cp_prev;
    char_u	*cp_str;	// matched text
    char_u	*(cp_text[CPT_COUNT]);	// text for the menu
#ifdef FEAT_EVAL
    typval_T	cp_user_data;
#endif
    char_u	*cp_fname;	// file containing the match, allocated when
				// cp_flags has CP_FREE_FNAME
    int		cp_flags;	// CP_ values
    int		cp_number;	// sequence number
}

static int	  compl_enter_selects = FALSE;

static int	  compl_get_longest = FALSE;

static int	  compl_no_insert = FALSE;

static int	  compl_no_select = FALSE;

static int	  compl_used_match;

static int	  compl_was_interrupted = FALSE;

static int	  compl_interrupted = FALSE;

static int	  compl_restarting = FALSE;

static int	  compl_started = FALSE;

static int	  ctrl_x_mode = CTRL_X_NORMAL;

static int	  compl_matches = 0;

static int	  compl_direction = FORWARD;

static int	  compl_shows_dir = FORWARD;

static int	  compl_pending = 0;

static int	  compl_length = 0;

static int	  compl_cont_mode = 0;

static int	  compl_cont_status = 0;

# define CONT_LOCAL	32	// for ctrl_x_mode 0, ^X^P/^X^N do a local

static int	  compl_opt_refresh_always = FALSE;

static int	  compl_opt_suppress_empty = FALSE;

static int ins_compl_add(char_u *str, int len, char_u *fname, char_u **cptext, typval_T *user_data, int cdir, int flags, int adup);

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static int  ins_compl_key2dir(int c);

static int  ins_compl_pum_key(int c);

static int  ins_compl_key2count(int c);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static unsigned  quote_meta(char_u *dest, char_u *str, int len);

static void spell_back_to_badword(void);

static int  spell_bad_len = 0;

static int compl_match_arraysize;


spell_back_to_badword(void)
{
    pos_T	tpos = curwin->w_cursor;

    spell_bad_len = spell_move_to(curwin, BACKWARD, TRUE, TRUE, NULL);
    if (curwin->w_cursor.col != tpos.col)
	start_arrow(&tpos);
}

static char *ctrl_x_msgs[] =
{
    N_(" Keyword completion (^N^P)"), // CTRL_X_NORMAL, ^P/^N compl.
    N_(" ^X mode (^]^D^E^F^I^K^L^N^O^Ps^U^V^Y)"),
    NULL, // CTRL_X_SCROLL: depends on state
    N_(" Whole line completion (^L^N^P)"),
    N_(" File name completion (^F^N^P)"),
    N_(" Tag completion (^]^N^P)"),
    N_(" Path pattern completion (^N^P)"),
    N_(" Definition completion (^D^N^P)"),
    NULL, // CTRL_X_FINISHED
    N_(" Dictionary completion (^K^N^P)"),
    N_(" Thesaurus completion (^T^N^P)"),
    N_(" Command-line completion (^V^N^P)"),
    N_(" User defined completion (^U^N^P)"),
    N_(" Omni completion (^O^N^P)"),
    N_(" Spelling suggestion (s^N^P)"),
    N_(" Keyword Local completion (^N^P)"),
    NULL,   // CTRL_X_EVAL doesn't use msg.
    N_(" Command-line completion (^V^N^P)"),
};

static char *ctrl_x_mode_names[] = {
	"keyword",
	"ctrl_x",
	"scroll",
	"whole_line",
	"files",
	"tags",
	"path_patterns",
	"path_defines",
	"unknown",	    // CTRL_X_FINISHED
	"dictionary",
	"thesaurus",
	"cmdline",
	"function",
	"omni",
	"spell",
	NULL,		    // CTRL_X_LOCAL_MSG only used in "ctrl_x_msgs"
	"eval",
	"cmdline",
};

# define CP_EQUAL	    8	// ins_compl_equal() always returns TRUE

static compl_T    *compl_first_match = NULL;

static compl_T    *compl_curr_match = NULL;

static compl_T    *compl_shown_match = NULL;

static compl_T    *compl_old_match = NULL;

static char_u	  *compl_leader = NULL;

static char_u	  *compl_pattern = NULL;

static pos_T	  compl_startpos;

static char_u	  *compl_orig_text = NULL;

static void ins_compl_longest_match(compl_T *match);

static void ins_compl_del_pum(void);

static void ins_compl_files(int count, char_u **files, int thesaurus, int flags, regmatch_T *regmatch, char_u *buf, int *dir);

static void ins_compl_free(void);

static int  ins_compl_need_restart(void);

static void ins_compl_new_leader(void);

static int  get_compl_len(void);

static void ins_compl_restart(void);

static void ins_compl_set_original_text(char_u *str);

static void ins_compl_fixRedoBufForLeader(char_u *ptr_arg);

static void ins_compl_add_list(list_T *list);

static void ins_compl_add_dict(dict_T *dict);

static void show_pum(int prev_w_wrow, int prev_w_leftcol);

static void spell_back_to_badword(void);

static int  spell_bad_len = 0;

static pumitem_T *compl_match_array = NULL;