int recv_files(int f_in, int f_out, char *local_name)
{
	int fd1,fd2;
	STRUCT_STAT st;
	int iflags, xlen;
	char *fname, fbuf[MAXPATHLEN];
	char xname[MAXPATHLEN];
	char fnametmp[MAXPATHLEN];
	char *fnamecmp, *partialptr;
	char fnamecmpbuf[MAXPATHLEN];
	uchar fnamecmp_type;
	struct file_struct *file;
	int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i;
	enum logcode log_code = log_before_transfer ? FLOG : FINFO;
	int max_phase = protocol_version >= 29 ? 2 : 1;
	int dflt_perms = (ACCESSPERMS & ~orig_umask);
#ifdef SUPPORT_ACLS
	const char *parent_dirname = "";
#endif
	int ndx, recv_ok;

	if (DEBUG_GTE(RECV, 1))
		rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used);

	if (delay_updates)
		delayed_bits = bitbag_create(cur_flist->used + 1);

	while (1) {
		cleanup_disable();

		/* This call also sets cur_flist. */
		ndx = read_ndx_and_attrs(f_in, f_out, &iflags, &fnamecmp_type,
					 xname, &xlen);
		if (ndx == NDX_DONE) {
			if (!am_server && INFO_GTE(PROGRESS, 2) && cur_flist) {
				set_current_file_index(NULL, 0);
				end_progress(0);
			}
			if (inc_recurse && first_flist) {
				if (read_batch) {
					ndx = first_flist->used + first_flist->ndx_start;
					gen_wants_ndx(ndx, first_flist->flist_num);
				}
				flist_free(first_flist);
				if (first_flist)
					continue;
			} else if (read_batch && first_flist) {
				ndx = first_flist->used;
				gen_wants_ndx(ndx, first_flist->flist_num);
			}
			if (++phase > max_phase)
				break;
			if (DEBUG_GTE(RECV, 1))
				rprintf(FINFO, "recv_files phase=%d\n", phase);
			if (phase == 2 && delay_updates)
				handle_delayed_updates(local_name);
			write_int(f_out, NDX_DONE);
			continue;
		}

		if (ndx - cur_flist->ndx_start >= 0)
			file = cur_flist->files[ndx - cur_flist->ndx_start];
		else
			file = dir_flist->files[cur_flist->parent_ndx];
		fname = local_name ? local_name : f_name(file, fbuf);

		if (daemon_filter_list.head
		    && check_filter(&daemon_filter_list, FLOG, fname, 0) < 0) {
			rprintf(FERROR, "attempt to hack rsync failed.\n");
			exit_cleanup(RERR_PROTOCOL);
		}

		if (DEBUG_GTE(RECV, 1))
			rprintf(FINFO, "recv_files(%s)\n", fname);

#ifdef SUPPORT_XATTRS
		if (preserve_xattrs && iflags & ITEM_REPORT_XATTR && do_xfers
		 && !(want_xattr_optim && BITS_SET(iflags, ITEM_XNAME_FOLLOWS|ITEM_LOCAL_CHANGE)))
			recv_xattr_request(file, f_in);
#endif

		if (!(iflags & ITEM_TRANSFER)) {
			maybe_log_item(file, iflags, itemizing, xname);
#ifdef SUPPORT_XATTRS
			if (preserve_xattrs && iflags & ITEM_REPORT_XATTR && do_xfers
			 && !BITS_SET(iflags, ITEM_XNAME_FOLLOWS|ITEM_LOCAL_CHANGE))
				set_file_attrs(fname, file, NULL, fname, 0);
#endif
			if (iflags & ITEM_IS_NEW) {
				stats.created_files++;
				if (S_ISREG(file->mode)) {
					/* Nothing further to count. */
				} else if (S_ISDIR(file->mode))
					stats.created_dirs++;
#ifdef SUPPORT_LINKS
				else if (S_ISLNK(file->mode))
					stats.created_symlinks++;
#endif
				else if (IS_DEVICE(file->mode))
					stats.created_devices++;
				else
					stats.created_specials++;
			}
			continue;
		}
		if (phase == 2) {
			rprintf(FERROR,
				"got transfer request in phase 2 [%s]\n",
				who_am_i());
			exit_cleanup(RERR_PROTOCOL);
		}

		if (file->flags & FLAG_FILE_SENT) {
			if (csum_length == SHORT_SUM_LENGTH) {
				if (keep_partial && !partial_dir)
					make_backups = -make_backups; /* prevents double backup */
				if (append_mode)
					sparse_files = -sparse_files;
				append_mode = -append_mode;
				csum_length = SUM_LENGTH;
				redoing = 1;
			}
		} else {
			if (csum_length != SHORT_SUM_LENGTH) {
				if (keep_partial && !partial_dir)
					make_backups = -make_backups;
				if (append_mode)
					sparse_files = -sparse_files;
				append_mode = -append_mode;
				csum_length = SHORT_SUM_LENGTH;
				redoing = 0;
			}
			if (iflags & ITEM_IS_NEW)
				stats.created_files++;
		}

		if (!am_server && INFO_GTE(PROGRESS, 1))
			set_current_file_index(file, ndx);
		stats.xferred_files++;
		stats.total_transferred_size += F_LENGTH(file);

		cleanup_got_literal = 0;

		if (read_batch) {
			int wanted = redoing
				   ? we_want_redo(ndx)
				   : gen_wants_ndx(ndx, cur_flist->flist_num);
			if (!wanted) {
				rprintf(FINFO,
					"(Skipping batched update for%s \"%s\")\n",
					redoing ? " resend of" : "",
					fname);
				discard_receive_data(f_in, F_LENGTH(file));
				file->flags |= FLAG_FILE_SENT;
				continue;
			}
		}

		remember_initial_stats();

		if (!do_xfers) { /* log the transfer */
			log_item(FCLIENT, file, iflags, NULL);
			if (read_batch)
				discard_receive_data(f_in, F_LENGTH(file));
			continue;
		}
		if (write_batch < 0) {
			log_item(FCLIENT, file, iflags, NULL);
			if (!am_server)
				discard_receive_data(f_in, F_LENGTH(file));
			if (inc_recurse)
				send_msg_int(MSG_SUCCESS, ndx);
			continue;
		}

		partialptr = partial_dir ? partial_dir_fname(fname) : fname;

		if (protocol_version >= 29) {
			switch (fnamecmp_type) {
			case FNAMECMP_FNAME:
				fnamecmp = fname;
				break;
			case FNAMECMP_PARTIAL_DIR:
				fnamecmp = partialptr;
				break;
			case FNAMECMP_BACKUP:
				fnamecmp = get_backup_name(fname);
				break;
			case FNAMECMP_FUZZY:
				if (file->dirname) {
					pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, file->dirname, xname);
					fnamecmp = fnamecmpbuf;
				} else
					fnamecmp = xname;
				break;
			default:
				if (fnamecmp_type > FNAMECMP_FUZZY && fnamecmp_type-FNAMECMP_FUZZY <= basis_dir_cnt) {
					fnamecmp_type -= FNAMECMP_FUZZY + 1;
					if (file->dirname) {
						stringjoin(fnamecmpbuf, sizeof fnamecmpbuf,
							   basis_dir[fnamecmp_type], "/", file->dirname, "/", xname, NULL);
					} else
						pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, basis_dir[fnamecmp_type], xname);
				} else if (fnamecmp_type >= basis_dir_cnt) {
					rprintf(FERROR,
						"invalid basis_dir index: %d.\n",
						fnamecmp_type);
					exit_cleanup(RERR_PROTOCOL);
				} else
					pathjoin(fnamecmpbuf, sizeof fnamecmpbuf, basis_dir[fnamecmp_type], fname);
				fnamecmp = fnamecmpbuf;
				break;
			}
			if (!fnamecmp || (daemon_filter_list.head
			  && check_filter(&daemon_filter_list, FLOG, fnamecmp, 0) < 0)) {
				fnamecmp = fname;
				fnamecmp_type = FNAMECMP_FNAME;
			}
		} else {
			/* Reminder: --inplace && --partial-dir are never
			 * enabled at the same time. */
			if (inplace && make_backups > 0) {
				if (!(fnamecmp = get_backup_name(fname)))
					fnamecmp = fname;
				else
					fnamecmp_type = FNAMECMP_BACKUP;
			} else if (partial_dir && partialptr)
				fnamecmp = partialptr;
			else
				fnamecmp = fname;
		}

		/* open the file */
		fd1 = do_open(fnamecmp, O_RDONLY, 0);

		if (fd1 == -1 && protocol_version < 29) {
			if (fnamecmp != fname) {
				fnamecmp = fname;
				fd1 = do_open(fnamecmp, O_RDONLY, 0);
			}

			if (fd1 == -1 && basis_dir[0]) {
				/* pre-29 allowed only one alternate basis */
				pathjoin(fnamecmpbuf, sizeof fnamecmpbuf,
					 basis_dir[0], fname);
				fnamecmp = fnamecmpbuf;
				fd1 = do_open(fnamecmp, O_RDONLY, 0);
			}
		}

		updating_basis_or_equiv = inplace
		    && (fnamecmp == fname || fnamecmp_type == FNAMECMP_BACKUP);

		if (fd1 == -1) {
			st.st_mode = 0;
			st.st_size = 0;
		} else if (do_fstat(fd1,&st) != 0) {
			rsyserr(FERROR_XFER, errno, "fstat %s failed",
				full_fname(fnamecmp));
			discard_receive_data(f_in, F_LENGTH(file));
			close(fd1);
			if (inc_recurse)
				send_msg_int(MSG_NO_SEND, ndx);
			continue;
		}

		if (fd1 != -1 && S_ISDIR(st.st_mode) && fnamecmp == fname) {
			/* this special handling for directories
			 * wouldn't be necessary if robust_rename()
			 * and the underlying robust_unlink could cope
			 * with directories
			 */
			rprintf(FERROR_XFER, "recv_files: %s is a directory\n",
				full_fname(fnamecmp));
			discard_receive_data(f_in, F_LENGTH(file));
			close(fd1);
			if (inc_recurse)
				send_msg_int(MSG_NO_SEND, ndx);
			continue;
		}

		if (fd1 != -1 && !S_ISREG(st.st_mode)) {
			close(fd1);
			fd1 = -1;
		}

		/* If we're not preserving permissions, change the file-list's
		 * mode based on the local permissions and some heuristics. */
		if (!preserve_perms) {
			int exists = fd1 != -1;
#ifdef SUPPORT_ACLS
			const char *dn = file->dirname ? file->dirname : ".";
			if (parent_dirname != dn
			 && strcmp(parent_dirname, dn) != 0) {
				dflt_perms = default_perms_for_dir(dn);
				parent_dirname = dn;
			}
#endif
			file->mode = dest_mode(file->mode, st.st_mode,
					       dflt_perms, exists);
		}

		/* We now check to see if we are writing the file "inplace" */
		if (inplace)  {
			fd2 = do_open(fname, O_WRONLY|O_CREAT, 0600);
			if (fd2 == -1) {
				rsyserr(FERROR_XFER, errno, "open %s failed",
					full_fname(fname));
			} else if (updating_basis_or_equiv)
				cleanup_set(NULL, NULL, file, fd1, fd2);
		} else {
			fd2 = open_tmpfile(fnametmp, fname, file);
			if (fd2 != -1)
				cleanup_set(fnametmp, partialptr, file, fd1, fd2);
		}

		if (fd2 == -1) {
			discard_receive_data(f_in, F_LENGTH(file));
			if (fd1 != -1)
				close(fd1);
			if (inc_recurse)
				send_msg_int(MSG_NO_SEND, ndx);
			continue;
		}

		/* log the transfer */
		if (log_before_transfer)
			log_item(FCLIENT, file, iflags, NULL);
		else if (!am_server && INFO_GTE(NAME, 1) && INFO_EQ(PROGRESS, 1))
			rprintf(FINFO, "%s\n", fname);

		/* recv file data */
		recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
				       fname, fd2, F_LENGTH(file));

		log_item(log_code, file, iflags, NULL);

		if (fd1 != -1)
			close(fd1);
		if (close(fd2) < 0) {
			rsyserr(FERROR, errno, "close failed on %s",
				full_fname(fnametmp));
			exit_cleanup(RERR_FILEIO);
		}

		if ((recv_ok && (!delay_updates || !partialptr)) || inplace) {
			if (partialptr == fname)
				partialptr = NULL;
			if (!finish_transfer(fname, fnametmp, fnamecmp,
					     partialptr, file, recv_ok, 1))
				recv_ok = -1;
			else if (fnamecmp == partialptr) {
				do_unlink(partialptr);
				handle_partial_dir(partialptr, PDIR_DELETE);
			}
		} else if (keep_partial && partialptr) {
			if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
				rprintf(FERROR,
				    "Unable to create partial-dir for %s -- discarding %s.\n",
				    local_name ? local_name : f_name(file, NULL),
				    recv_ok ? "completed file" : "partial file");
				do_unlink(fnametmp);
				recv_ok = -1;
			} else if (!finish_transfer(partialptr, fnametmp, fnamecmp, NULL,
						    file, recv_ok, !partial_dir))
				recv_ok = -1;
			else if (delay_updates && recv_ok) {
				bitbag_set_bit(delayed_bits, ndx);
				recv_ok = 2;
			} else
				partialptr = NULL;
		} else
			do_unlink(fnametmp);

		cleanup_disable();

		if (read_batch)
			file->flags |= FLAG_FILE_SENT;

		switch (recv_ok) {
		case 2:
			break;
		case 1:
			if (remove_source_files || inc_recurse
			 || (preserve_hard_links && F_IS_HLINKED(file)))
				send_msg_int(MSG_SUCCESS, ndx);
			break;
		case 0: {
			enum logcode msgtype = redoing ? FERROR_XFER : FWARNING;
			if (msgtype == FERROR_XFER || INFO_GTE(NAME, 1)) {
				char *errstr, *redostr, *keptstr;
				if (!(keep_partial && partialptr) && !inplace)
					keptstr = "discarded";
				else if (partial_dir)
					keptstr = "put into partial-dir";
				else
					keptstr = "retained";
				if (msgtype == FERROR_XFER) {
					errstr = "ERROR";
					redostr = "";
				} else {
					errstr = "WARNING";
					redostr = read_batch ? " (may try again)"
							     : " (will try again)";
				}
				rprintf(msgtype,
					"%s: %s failed verification -- update %s%s.\n",
					errstr, local_name ? f_name(file, NULL) : fname,
					keptstr, redostr);
			}
			if (!redoing) {
				if (read_batch)
					flist_ndx_push(&batch_redo_list, ndx);
				send_msg_int(MSG_REDO, ndx);
				file->flags |= FLAG_FILE_SENT;
			} else if (inc_recurse)
				send_msg_int(MSG_NO_SEND, ndx);
			break;
		    }
		case -1:
			if (inc_recurse)
				send_msg_int(MSG_NO_SEND, ndx);
			break;
		}
	}
	if (make_backups < 0)
		make_backups = -make_backups;

	if (phase == 2 && delay_updates) /* for protocol_version < 29 */
		handle_delayed_updates(local_name);

	if (DEBUG_GTE(RECV, 1))
		rprintf(FINFO,"recv_files finished\n");

	return 0;
}


static void discard_receive_data(int f_in, OFF_T length)
{
	receive_data(f_in, NULL, -1, 0, NULL, -1, length);
}

Check daemon filter against fnamecmp in recv_files().

extern int dry_run;

extern int do_xfers;

extern int am_root;

extern int am_server;

extern int inc_recurse;

extern int log_before_transfer;

extern int stdout_format_has_i;

extern int logfile_format_has_i;

extern int want_xattr_optim;

extern int csum_length;

extern int read_batch;

extern int write_batch;

extern int batch_gen_fd;

extern int protocol_version;

extern int relative_paths;

extern int preserve_hard_links;

extern int preserve_perms;

extern int preserve_xattrs;

extern int basis_dir_cnt;

extern int make_backups;

extern int cleanup_got_literal;

extern int remove_source_files;

extern int append_mode;

extern int sparse_files;

extern int preallocate_files;

extern int keep_partial;

extern int checksum_seed;

extern int whole_file;

extern int inplace;

extern int allowed_lull;

extern int delay_updates;

extern int xfersum_type;

extern OFF_T preallocated_len;

static struct bitbag *delayed_bits = NULL;

static int phase = 0, redoing = 0;

static flist_ndx_list batch_redo_list;

static int updating_basis_or_equiv;

#define TMPNAME_SUFFIX_LEN ((int)sizeof TMPNAME_SUFFIX - 1)


repodata_set(Repodata *data, Id solvid, Repokey *key, Id val)
{
  Id keyid;

  keyid = repodata_key2id(data, key, 1);
  repodata_insert_keyid(data, solvid, keyid, val, 1);
}

repodata_schema2id: fix heap-buffer-overflow in memcmp

When the length of last schema in data->schemadata is
less than length of input schema, we got a read overflow
in asan test.

Signed-off-by: Zhipeng Xie <xiezhipeng1@huawei.com>

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}


repodata_internalize(Repodata *data)
{
  Repokey *key, solvkey;
  Id entry, nentry;
  Id schemaid, keyid, *schema, *sp, oldschemaid, *keyp, *seen;
  Offset *oldincoreoffs = 0;
  int schemaidx;
  unsigned char *dp, *ndp;
  int neednewschema;
  struct extdata newincore;
  struct extdata newvincore;
  Id solvkeyid;
  Id *keylink;
  int haveoldkl;

  if (!data->attrs && !data->xattrs)
    return;

#if 0
  printf("repodata_internalize %d\n", data->repodataid);
  printf("  attr data: %d K\n", data->attrdatalen / 1024);
  printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
#endif
  newvincore.buf = data->vincore;
  newvincore.len = data->vincorelen;

  /* find the solvables key, create if needed */
  memset(&solvkey, 0, sizeof(solvkey));
  solvkey.name = REPOSITORY_SOLVABLES;
  solvkey.type = REPOKEY_TYPE_FLEXARRAY;
  solvkey.size = 0;
  solvkey.storage = KEY_STORAGE_INCORE;
  solvkeyid = repodata_key2id(data, &solvkey, data->end != data->start ? 1 : 0);

  schema = solv_malloc2(data->nkeys, sizeof(Id));
  seen = solv_malloc2(data->nkeys, sizeof(Id));

  /* Merge the data already existing (in data->schemata, ->incoredata and
     friends) with the new attributes in data->attrs[].  */
  nentry = data->end - data->start;
  memset(&newincore, 0, sizeof(newincore));
  data_addid(&newincore, 0);	/* start data at offset 1 */

  data->mainschema = 0;
  data->mainschemaoffsets = solv_free(data->mainschemaoffsets);

  keylink = calculate_keylink(data);
  /* join entry data */
  /* we start with the meta data, entry -1 */
  for (entry = -1; entry < nentry; entry++)
    {
      oldschemaid = 0;
      dp = data->incoredata;
      if (dp)
	{
	  dp += entry >= 0 ? data->incoreoffset[entry] : 1;
          dp = data_read_id(dp, &oldschemaid);
	}
      memset(seen, 0, data->nkeys * sizeof(Id));
#if 0
fprintf(stderr, "oldschemaid %d\n", oldschemaid);
fprintf(stderr, "schemata %d\n", data->schemata[oldschemaid]);
fprintf(stderr, "schemadata %p\n", data->schemadata);
#endif

      /* seen: -1: old data,  0: skipped,  >0: id + 1 */
      neednewschema = 0;
      sp = schema;
      haveoldkl = 0;
      for (keyp = data->schemadata + data->schemata[oldschemaid]; *keyp; keyp++)
	{
	  if (seen[*keyp])
	    {
	      /* oops, should not happen */
	      neednewschema = 1;
	      continue;
	    }
	  seen[*keyp] = -1;	/* use old marker */
	  *sp++ = *keyp;
	  if (keylink[*keyp])
	    haveoldkl = 1;	/* potential keylink conflict */
	}

      /* strip solvables key */
      if (entry < 0 && solvkeyid && seen[solvkeyid])
	{
	  *sp = 0;
	  for (sp = keyp = schema; *sp; sp++)
	    if (*sp != solvkeyid)
	      *keyp++ = *sp;
	  sp = keyp;
	  seen[solvkeyid] = 0;
	  neednewschema = 1;
	}

      /* add new entries */
      if (entry >= 0)
	keyp = data->attrs ? data->attrs[entry] : 0;
      else
        keyp = data->xattrs ? data->xattrs[1] : 0;
      if (keyp)
        for (; *keyp; keyp += 2)
	  {
	    if (!seen[*keyp])
	      {
	        neednewschema = 1;
	        *sp++ = *keyp;
		if (haveoldkl && keylink[*keyp])		/* this should be pretty rare */
		  {
		    Id kl;
		    for (kl = keylink[*keyp]; kl != *keyp; kl = keylink[kl])
		      if (seen[kl] == -1)
		        {
			  /* replacing old key kl, remove from schema and seen */
			  Id *osp;
			  for (osp = schema; osp < sp; osp++)
			    if (*osp == kl)
			      {
			        memmove(osp, osp + 1, (sp - osp) * sizeof(Id));
			        sp--;
			        seen[kl] = 0;
				break;
			      }
		        }
		  }
	      }
	    seen[*keyp] = keyp[1] + 1;
	  }

      /* add solvables key if needed */
      if (entry < 0 && data->end != data->start)
	{
	  *sp++ = solvkeyid;	/* always last in schema */
	  neednewschema = 1;
	}

      /* commit schema */
      *sp = 0;
      if (neednewschema)
        /* Ideally we'd like to sort the new schema here, to ensure
	   schema equality independend of the ordering. */
	schemaid = repodata_schema2id(data, schema, 1);
      else
	schemaid = oldschemaid;

      if (entry < 0)
	{
	  data->mainschemaoffsets = solv_calloc(sp - schema, sizeof(Id));
	  data->mainschema = schemaid;
	}

      /* find offsets in old incore data */
      if (oldschemaid)
	{
	  Id *lastneeded = 0;
	  for (sp = data->schemadata + data->schemata[oldschemaid]; *sp; sp++)
	    if (seen[*sp] == -1)
	      lastneeded = sp + 1;
	  if (lastneeded)
	    {
	      if (!oldincoreoffs)
	        oldincoreoffs = solv_malloc2(data->nkeys, 2 * sizeof(Offset));
	      for (sp = data->schemadata + data->schemata[oldschemaid]; sp != lastneeded; sp++)
		{
		  /* Skip the data associated with this old key.  */
		  key = data->keys + *sp;
		  ndp = dp;
		  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
		    {
		      ndp = data_skip(ndp, REPOKEY_TYPE_ID);
		      ndp = data_skip(ndp, REPOKEY_TYPE_ID);
		    }
		  else if (key->storage == KEY_STORAGE_INCORE)
		    ndp = data_skip_key(data, ndp, key);
		  oldincoreoffs[*sp * 2] = dp - data->incoredata;
		  oldincoreoffs[*sp * 2 + 1] = ndp - dp;
		  dp = ndp;
		}
	    }
	}

      /* just copy over the complete old entry (including the schemaid) if there was no new data */
      if (entry >= 0 && !neednewschema && oldschemaid && (!data->attrs || !data->attrs[entry]) && dp)
	{
	  ndp = data->incoredata + data->incoreoffset[entry];
	  data->incoreoffset[entry] = newincore.len;
	  data_addblob(&newincore, ndp, dp - ndp);
	  goto entrydone;
	}

      /* Now create data blob.  We walk through the (possibly new) schema
	 and either copy over old data, or insert the new.  */
      if (entry >= 0)
        data->incoreoffset[entry] = newincore.len;
      data_addid(&newincore, schemaid);

      /* we don't use a pointer to the schemadata here as repodata_serialize_key
       * may call repodata_schema2id() which might realloc our schemadata */
      for (schemaidx = data->schemata[schemaid]; (keyid = data->schemadata[schemaidx]) != 0; schemaidx++)
	{
	  if (entry < 0)
	    {
	      data->mainschemaoffsets[schemaidx - data->schemata[schemaid]] = newincore.len;
	      if (keyid == solvkeyid)
		{
		  /* add flexarray entry count */
		  data_addid(&newincore, data->end - data->start);
		  break;	/* always the last entry */
		}
	    }
	  if (seen[keyid] == -1)
	    {
	      if (oldincoreoffs[keyid * 2 + 1])
		data_addblob(&newincore, data->incoredata + oldincoreoffs[keyid * 2], oldincoreoffs[keyid * 2 + 1]);
	    }
	  else if (seen[keyid])
	    repodata_serialize_key(data, &newincore, &newvincore, schema, data->keys + keyid, seen[keyid] - 1);
	}

entrydone:
      /* free memory */
      if (entry >= 0 && data->attrs)
	{
	  if (data->attrs[entry])
	    data->attrs[entry] = solv_free(data->attrs[entry]);
	  if (entry && entry % 4096 == 0 && data->nxattrs <= 2 && entry + 64 < nentry)
	    {
	      compact_attrdata(data, entry + 1, nentry);	/* try to free some memory */
#if 0
	      printf("  attr data: %d K\n", data->attrdatalen / 1024);
	      printf("  attrid data: %d K\n", data->attriddatalen / (1024 / 4));
	      printf("  incore data: %d K\n", newincore.len / 1024);
	      printf("  sum: %d K\n", (newincore.len + data->attrdatalen + data->attriddatalen * 4) / 1024);
	      /* malloc_stats(); */
#endif
	    }
	}
    }
  /* free all xattrs */
  for (entry = 0; entry < data->nxattrs; entry++)
    if (data->xattrs[entry])
      solv_free(data->xattrs[entry]);
  data->xattrs = solv_free(data->xattrs);
  data->nxattrs = 0;

  data->lasthandle = 0;
  data->lastkey = 0;
  data->lastdatalen = 0;
  solv_free(schema);
  solv_free(seen);
  solv_free(keylink);
  solv_free(oldincoreoffs);
  repodata_free_schemahash(data);

  solv_free(data->incoredata);
  data->incoredata = newincore.buf;
  data->incoredatalen = newincore.len;
  data->incoredatafree = 0;

  data->vincore = newvincore.buf;
  data->vincorelen = newvincore.len;

  data->attrs = solv_free(data->attrs);
  data->attrdata = solv_free(data->attrdata);
  data->attriddata = solv_free(data->attriddata);
  data->attrnum64data = solv_free(data->attrnum64data);
  data->attrdatalen = 0;
  data->attriddatalen = 0;
  data->attrnum64datalen = 0;
#if 0
  printf("repodata_internalize %d done\n", data->repodataid);
  printf("  incore data: %d K\n", data->incoredatalen / 1024);
#endif
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_new_handle(Repodata *data)
{
  if (!data->nxattrs)
    {
      data->xattrs = solv_calloc_block(1, sizeof(Id *), REPODATA_BLOCK);
      data->nxattrs = 2;	/* -1: SOLVID_META */
    }
  data->xattrs = solv_extend(data->xattrs, data->nxattrs, 1, sizeof(Id *), REPODATA_BLOCK);
  data->xattrs[data->nxattrs] = 0;
  return -(data->nxattrs++);
}

#define REPODATA_BLOCK 255

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);


repodata_lookup_num(Repodata *data, Id solvid, Id keyname, unsigned long long notfound)
{
  unsigned char *dp;
  Repokey *key;
  unsigned int high, low;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return notfound;
  switch (key->type)
    {
    case REPOKEY_TYPE_NUM:
      data_read_num64(dp, &low, &high);
      return (unsigned long long)high << 32 | low;
    case REPOKEY_TYPE_CONSTANT:
      return key->size;
    default:
      return notfound;
    }
}


repodata_search_keyskip(Repodata *data, Id solvid, Id keyname, int flags, Id *keyskip, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  Id schema;
  Repokey *key;
  Id keyid, *kp, *keyp;
  unsigned char *dp, *ddp;
  int onekey = 0;
  int stop;
  KeyValue kv;
  Solvable *s;

  if (!maybe_load_repodata(data, keyname))
    return;
  if ((flags & SEARCH_SUBSCHEMA) != 0)
    {
      flags ^= SEARCH_SUBSCHEMA;
      kv.parent = (KeyValue *)keyskip;
      keyskip = 0;
      schema = kv.parent->id;
      dp = (unsigned char *)kv.parent->str;
    }
  else
    {
      schema = 0;
      dp = solvid2data(data, solvid, &schema);
      if (!dp)
	return;
      kv.parent = 0;
    }
  s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
  keyp = data->schemadata + data->schemata[schema];
  if (keyname)
    {
      /* search for a specific key */
      for (kp = keyp; *kp; kp++)
	if (data->keys[*kp].name == keyname)
	  break;
      if (!*kp)
	return;
      dp = forward_to_key(data, *kp, keyp, dp);
      if (!dp)
	return;
      keyp = kp;
      onekey = 1;
    }
  while ((keyid = *keyp++) != 0)
    {
      stop = 0;
      key = data->keys + keyid;
      ddp = get_data(data, key, &dp, *keyp && !onekey ? 1 : 0);

      if (keyskip && (key->name >= keyskip[0] || keyskip[3 + key->name] != keyskip[1] + data->repodataid))
	{
	  if (onekey)
	    return;
	  continue;
	}
      if (key->type == REPOKEY_TYPE_DELETED && !(flags & SEARCH_KEEP_TYPE_DELETED))
	{
	  if (onekey)
	    return;
	  continue;
	}
      if (key->type == REPOKEY_TYPE_FLEXARRAY || key->type == REPOKEY_TYPE_FIXARRAY)
	{
	  kv.entry = -1;
	  ddp = data_read_id(ddp, (Id *)&kv.num);
	  kv.str = (const char *)ddp;
	  stop = repodata_search_array(data, solvid, 0, flags, key, &kv, callback, cbdata);
	  if (onekey || stop > SEARCH_NEXT_KEY)
	    return;
	  continue;
	}
      kv.entry = 0;
      do
	{
	  ddp = data_fetch(ddp, &kv, key);
	  if (!ddp)
	    break;
	  stop = callback(cbdata, s, data, key, &kv);
	  kv.entry++;
	}
      while (!kv.eof && !stop);
      if (onekey || stop > SEARCH_NEXT_KEY)
	return;
    }
}


repodata_set_strn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
{
  if (!str[l])
    repodata_set_str(data, solvid, keyname, str);
  else
    {
      char *s = solv_strdup(str);
      s[l] = 0;
      repodata_set_str(data, solvid, keyname, s);
      free(s);
    }
}


dataiterator_free(Dataiterator *di)
{
  if (di->matcher.match)
    datamatcher_free(&di->matcher);
  if (di->dupstr)
    solv_free(di->dupstr);
  if (di->oldkeyskip)
    solv_free(di->oldkeyskip);
}

#define _GNU_SOURCE

#define REPODATA_BLOCK 255

#define SCHEMATA_BLOCK 31

#define SCHEMATADATA_BLOCK 255

#define REPODATA_ATTRS_BLOCK 31

#define EXTDATA_BLOCK 1023

#define REPODATA_ATTRIDDATA_BLOCK 63

#define REPODATA_ATTRDATA_BLOCK 1023

#define REPODATA_ATTRNUM64DATA_BLOCK 15

#define DIRCACHE_SIZE 41	/* < 1k */

struct extdata {
  unsigned char *buf;
  int len;
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}


repodata_set_bin_checksum(Repodata *data, Id solvid, Id keyname, Id type,
		      const unsigned char *str)
{
  Repokey key;
  int l;

  if (!(l = solv_chksum_len(type)))
    return;
  key.name = keyname;
  key.type = type;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
  memcpy(data->attrdata + data->attrdatalen, str, l);
  repodata_set(data, solvid, &key, data->attrdatalen);
  data->attrdatalen += l;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

#define REPODATA_ATTRDATA_BLOCK 1023

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_set_binary(Repodata *data, Id solvid, Id keyname, void *buf, int len)
{
  Repokey key;
  unsigned char *dp;

  if (len < 0)
    return;
  key.name = keyname;
  key.type = REPOKEY_TYPE_BINARY;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  data->attrdata = solv_extend(data->attrdata, data->attrdatalen, len + 5, 1, REPODATA_ATTRDATA_BLOCK);
  dp = data->attrdata + data->attrdatalen;
  if (len >= (1 << 14))
    {
      if (len >= (1 << 28))
        *dp++ = (len >> 28) | 128;
      if (len >= (1 << 21))
        *dp++ = (len >> 21) | 128;
      *dp++ = (len >> 14) | 128;
    }
  if (len >= (1 << 7))
    *dp++ = (len >> 7) | 128;
  *dp++ = len & 127;
  if (len)
    memcpy(dp, buf, len);
  repodata_set(data, solvid, &key, data->attrdatalen);
  data->attrdatalen = dp + len - data->attrdata;
}


repodata_merge_some_attrs(Repodata *data, Id dest, Id src, Map *keyidmap, int overwrite)
{
  Id *keyp;
  if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
    return;
  for (; *keyp; keyp += 2)
    if (!keyidmap || MAPTST(keyidmap, keyp[0]))
      repodata_insert_keyid(data, dest, keyp[0], keyp[1], overwrite);
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct extdata {
  unsigned char *buf;
  int len;
}


datamatcher_init(Datamatcher *ma, const char *match, int flags)
{
  match = match ? solv_strdup(match) : 0;
  ma->match = match;
  ma->flags = flags;
  ma->error = 0;
  ma->matchdata = 0;
  if ((flags & SEARCH_STRINGMASK) == SEARCH_REGEX)
    {
      ma->matchdata = solv_calloc(1, sizeof(regex_t));
      ma->error = regcomp((regex_t *)ma->matchdata, match, REG_EXTENDED | REG_NOSUB | REG_NEWLINE | ((flags & SEARCH_NOCASE) ? REG_ICASE : 0));
      if (ma->error)
	{
	  solv_free(ma->matchdata);
	  ma->flags = (flags & ~SEARCH_STRINGMASK) | SEARCH_ERROR;
	}
    }
  if ((flags & SEARCH_FILES) != 0 && match)
    {
      /* prepare basename check */
      if ((flags & SEARCH_STRINGMASK) == SEARCH_STRING || (flags & SEARCH_STRINGMASK) == SEARCH_STRINGEND)
	{
	  const char *p = strrchr(match, '/');
	  ma->matchdata = (void *)(p ? p + 1 : match);
	}
      else if ((flags & SEARCH_STRINGMASK) == SEARCH_GLOB)
	{
	  const char *p;
	  for (p = match + strlen(match) - 1; p >= match; p--)
	    if (*p == '[' || *p == ']' || *p == '*' || *p == '?' || *p == '/')
	      break;
	  ma->matchdata = (void *)(p + 1);
	}
    }
  return ma->error;
}


repodata_free_dircache(Repodata *data)
{
  data->dircache = solv_free(data->dircache);
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}


repodata_free(Repodata *data)
{
  Repo *repo = data->repo;
  int i = data - repo->repodata;
  if (i == 0)
    return;
  repodata_freedata(data);
  if (i < repo->nrepodata - 1)
    {
      /* whoa! this changes the repodataids! */
      memmove(repo->repodata + i, repo->repodata + i + 1, (repo->nrepodata - 1 - i) * sizeof(Repodata));
      for (; i < repo->nrepodata - 1; i++)
	repo->repodata[i].repodataid = i;
    }
  repo->nrepodata--;
  if (repo->nrepodata == 1)
    {
      repo->repodata = solv_free(repo->repodata);
      repo->nrepodata = 0;
    }
}


repodata_insert_keyid(Repodata *data, Id handle, Id keyid, Id val, int overwrite)
{
  Id *pp;
  Id *ap, **app;
  int i;

  app = repodata_get_attrp(data, handle);
  ap = *app;
  i = 0;
  if (ap)
    {
      /* Determine equality based on the name only, allows us to change
         type (when overwrite is set), and makes TYPE_CONSTANT work.  */
      for (pp = ap; *pp; pp += 2)
        if (data->keys[*pp].name == data->keys[keyid].name)
          break;
      if (*pp)
        {
	  if (overwrite || data->keys[*pp].type == REPOKEY_TYPE_DELETED)
	    {
	      pp[0] = keyid;
              pp[1] = val;
	    }
          return;
        }
      i = pp - ap;
    }
  ap = solv_extend(ap, i, 3, sizeof(Id), REPODATA_ATTRS_BLOCK);
  *app = ap;
  pp = ap + i;
  *pp++ = keyid;
  *pp++ = val;
  *pp = 0;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

#define REPODATA_ATTRS_BLOCK 31

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_add_stub(Repodata **datap)
{
  Repodata *data = *datap;
  Repo *repo = data->repo;
  Id repodataid = data - repo->repodata;
  Repodata *sdata = repo_add_repodata(repo, 0);
  data = repo->repodata + repodataid;
  if (data->end > data->start)
    repodata_extend_block(sdata, data->start, data->end - data->start);
  sdata->state = REPODATA_STUB;
  sdata->loadcallback = repodata_stub_loader;
  *datap = data;
  return sdata;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);


repodata_add_dirstr(Repodata *data, Id solvid, Id keyname, Id dir, const char *str)
{
  Id stroff;
  int l;

  assert(dir);
  l = strlen(str) + 1;
  data->attrdata = solv_extend(data->attrdata, data->attrdatalen, l, 1, REPODATA_ATTRDATA_BLOCK);
  memcpy(data->attrdata + data->attrdatalen, str, l);
  stroff = data->attrdatalen;
  data->attrdatalen += l;

#if 0
fprintf(stderr, "repodata_add_dirstr %d %d %s (%d)\n", solvid, dir, str,  data->attriddatalen);
#endif
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRSTRARRAY, 2);
  data->attriddata[data->attriddatalen++] = dir;
  data->attriddata[data->attriddatalen++] = stroff;
  data->attriddata[data->attriddatalen++] = 0;
}


repodata_globalize_id(Repodata *data, Id id, int create)
{
  if (!id || !data || !data->localpool)
    return id;
  return pool_str2id(data->repo->pool, stringpool_id2str(&data->spool, id), create);
}


repodata_lookup_idarray(Repodata *data, Id solvid, Id keyname, Queue *q)
{
  unsigned char *dp;
  Repokey *key;
  Id id;
  int eof = 0;

  queue_empty(q);
  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return 0;
  switch (key->type)
    {
    case REPOKEY_TYPE_CONSTANTID:
      queue_push(q, key->size);
      break;
    case REPOKEY_TYPE_ID:
      dp = data_read_id(dp, &id);
      queue_push(q, id);
      break;
    case REPOKEY_TYPE_IDARRAY:
      for (;;)
	{
	  dp = data_read_ideof(dp, &id, &eof);
	  queue_push(q, id);
	  if (eof)
	    break;
	}
      break;
    default:
      return 0;
    }
  return 1;
}


repodata_stub_loader(Repodata *data)
{
  Repo *repo = data->repo;
  Pool *pool = repo->pool;
  int r, i;
  struct s_Pool_tmpspace oldtmpspace;
  Datapos oldpos;

  if (!pool->loadcallback)
    {
      data->state = REPODATA_ERROR;
      return;
    }
  data->state = REPODATA_LOADING;

  /* save tmp space and pos */
  oldtmpspace = pool->tmpspace;
  memset(&pool->tmpspace, 0, sizeof(pool->tmpspace));
  oldpos = pool->pos;

  r = pool->loadcallback(pool, data, pool->loadcallbackdata);

  /* restore tmp space and pos */
  for (i = 0; i < POOL_TMPSPACEBUF; i++)
    solv_free(pool->tmpspace.buf[i]);
  pool->tmpspace = oldtmpspace;
  if (r && oldpos.repo == repo && oldpos.repodataid == data->repodataid)
    memset(&oldpos, 0, sizeof(oldpos));
  pool->pos = oldpos;

  data->state = r ? REPODATA_AVAILABLE : REPODATA_ERROR;
}


dataiterator_step(Dataiterator *di)
{
  Id schema;

  if (di->state == di_nextattr && di->key->storage == KEY_STORAGE_VERTICAL_OFFSET && di->vert_ddp && di->vert_storestate != di->data->storestate)
    {
      unsigned int ddpoff = di->ddp - di->vert_ddp;
      di->vert_off += ddpoff;
      di->vert_len -= ddpoff;
      di->ddp = di->vert_ddp = get_vertical_data(di->data, di->key, di->vert_off, di->vert_len);
      di->vert_storestate = di->data->storestate;
      if (!di->ddp)
	di->state = di_nextkey;
    }
  for (;;)
    {
      switch (di->state)
	{
	case di_enterrepo: di_enterrepo:
	  if (!di->repo || (di->repo->disabled && !(di->flags & SEARCH_DISABLED_REPOS)))
	    goto di_nextrepo;
	  if (!(di->flags & SEARCH_THISSOLVID))
	    {
	      di->solvid = di->repo->start - 1;	/* reset solvid iterator */
	      goto di_nextsolvable;
	    }
	  /* FALLTHROUGH */

	case di_entersolvable: di_entersolvable:
	  if (!di->repodataid)
	    goto di_enterrepodata;	/* POS case, repodata is set */
	  if (di->solvid > 0 && !(di->flags & SEARCH_NO_STORAGE_SOLVABLE) && (!di->keyname || (di->keyname >= SOLVABLE_NAME && di->keyname <= RPM_RPMDBID)) && di->nparents - di->rootlevel == di->nkeynames)
	    {
	      extern Repokey repo_solvablekeys[RPM_RPMDBID - SOLVABLE_NAME + 1];
	      di->key = repo_solvablekeys + (di->keyname ? di->keyname - SOLVABLE_NAME : 0);
	      di->data = 0;
	      goto di_entersolvablekey;
	    }

	  if (di->keyname)
	    {
	      di->data = di->keyname == SOLVABLE_FILELIST ? repo_lookup_filelist_repodata(di->repo, di->solvid, &di->matcher) : repo_lookup_repodata_opt(di->repo, di->solvid, di->keyname);
	      if (!di->data)
		goto di_nextsolvable;
	      di->repodataid = di->data - di->repo->repodata;
	      di->keyskip = 0;
	      goto di_enterrepodata;
	    }
	di_leavesolvablekey:
	  di->repodataid = 1;	/* reset repodata iterator */
	  di->keyskip = repo_create_keyskip(di->repo, di->solvid, &di->oldkeyskip);
	  /* FALLTHROUGH */

	case di_enterrepodata: di_enterrepodata:
	  if (di->repodataid)
	    {
	      if (di->repodataid >= di->repo->nrepodata)
		goto di_nextsolvable;
	      di->data = di->repo->repodata + di->repodataid;
	    }
	  if (!maybe_load_repodata(di->data, di->keyname))
	    goto di_nextrepodata;
	  di->dp = solvid2data(di->data, di->solvid, &schema);
	  if (!di->dp)
	    goto di_nextrepodata;
	  if (di->solvid == SOLVID_POS)
	    di->solvid = di->pool->pos.solvid;
	  /* reset key iterator */
	  di->keyp = di->data->schemadata + di->data->schemata[schema];
	  /* FALLTHROUGH */

	case di_enterschema: di_enterschema:
	  if (di->keyname)
	    di->dp = dataiterator_find_keyname(di, di->keyname);
	  if (!di->dp || !*di->keyp)
	    {
	      if (di->kv.parent)
		goto di_leavesub;
	      goto di_nextrepodata;
	    }
	  /* FALLTHROUGH */

	case di_enterkey: di_enterkey:
	  di->kv.entry = -1;
	  di->key = di->data->keys + *di->keyp;
	  if (!di->dp)
	    goto di_nextkey;
	  /* this is get_data() modified to store vert_ data */
	  if (di->key->storage == KEY_STORAGE_VERTICAL_OFFSET)
	    {
	      Id off, len;
	      di->dp = data_read_id(di->dp, &off);
	      di->dp = data_read_id(di->dp, &len);
	      di->vert_ddp = di->ddp = get_vertical_data(di->data, di->key, off, len);
	      di->vert_off = off;
	      di->vert_len = len;
	      di->vert_storestate = di->data->storestate;
	    }
	  else if (di->key->storage == KEY_STORAGE_INCORE)
	    {
	      di->ddp = di->dp;		/* start of data */
	      if (di->keyp[1] && (!di->keyname || (di->flags & SEARCH_SUB) != 0))
		di->dp = data_skip_key(di->data, di->dp, di->key);	/* advance to next key */
	    }
	  else
	    di->ddp = 0;
	  if (!di->ddp)
	    goto di_nextkey;
	  if (di->keyskip && (di->key->name >= di->keyskip[0] || di->keyskip[3 + di->key->name] != di->keyskip[1] + di->data->repodataid))
	    goto di_nextkey;
          if (di->key->type == REPOKEY_TYPE_DELETED && !(di->flags & SEARCH_KEEP_TYPE_DELETED))
	    goto di_nextkey;
	  if (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY)
	    goto di_enterarray;
	  if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
	    goto di_nextkey;
	  /* FALLTHROUGH */

	case di_nextattr:
          di->kv.entry++;
	  di->ddp = data_fetch(di->ddp, &di->kv, di->key);
	  di->state = di->kv.eof ? di_nextkey : di_nextattr;
	  break;

	case di_nextkey: di_nextkey:
	  if (!di->keyname && *++di->keyp)
	    goto di_enterkey;
	  if (di->kv.parent)
	    goto di_leavesub;
	  /* FALLTHROUGH */

	case di_nextrepodata: di_nextrepodata:
	  if (!di->keyname && di->repodataid && ++di->repodataid < di->repo->nrepodata)
	      goto di_enterrepodata;
	  /* FALLTHROUGH */

	case di_nextsolvable: di_nextsolvable:
	  if (!(di->flags & SEARCH_THISSOLVID))
	    {
	      if (di->solvid < 0)
		di->solvid = di->repo->start;
	      else
	        di->solvid++;
	      for (; di->solvid < di->repo->end; di->solvid++)
		{
		  if (di->pool->solvables[di->solvid].repo == di->repo)
		    goto di_entersolvable;
		}
	    }
	  /* FALLTHROUGH */

	case di_nextrepo: di_nextrepo:
	  if (di->repoid > 0)
	    {
	      di->repoid++;
	      di->repodataid = 1;
	      if (di->repoid < di->pool->nrepos)
		{
		  di->repo = di->pool->repos[di->repoid];
	          goto di_enterrepo;
		}
	    }
	/* FALLTHROUGH */

	case di_bye: di_bye:
	  di->state = di_bye;
	  return 0;

	case di_enterarray: di_enterarray:
	  if (di->key->name == REPOSITORY_SOLVABLES)
	    goto di_nextkey;
	  di->ddp = data_read_id(di->ddp, (Id *)&di->kv.num);
	  di->kv.eof = 0;
	  di->kv.entry = -1;
	  /* FALLTHROUGH */

	case di_nextarrayelement: di_nextarrayelement:
	  di->kv.entry++;
	  if (di->kv.entry)
	    di->ddp = data_skip_schema(di->data, di->ddp, di->kv.id);
	  if (di->kv.entry == di->kv.num)
	    {
	      if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
		goto di_nextkey;
	      if (!(di->flags & SEARCH_ARRAYSENTINEL))
		goto di_nextkey;
	      di->kv.str = (char *)di->ddp;
	      di->kv.eof = 2;
	      di->state = di_nextkey;
	      break;
	    }
	  if (di->kv.entry == di->kv.num - 1)
	    di->kv.eof = 1;
	  if (di->key->type == REPOKEY_TYPE_FLEXARRAY || !di->kv.entry)
	    di->ddp = data_read_id(di->ddp, &di->kv.id);
	  di->kv.str = (char *)di->ddp;
	  if (di->nkeynames && di->nparents - di->rootlevel < di->nkeynames)
	    goto di_entersub;
	  if ((di->flags & SEARCH_SUB) != 0)
	    di->state = di_entersub;
	  else
	    di->state = di_nextarrayelement;
	  break;

	case di_entersub: di_entersub:
	  if (di->nparents == sizeof(di->parents)/sizeof(*di->parents) - 1)
	    goto di_nextarrayelement;	/* sorry, full */
	  di->parents[di->nparents].kv = di->kv;
	  di->parents[di->nparents].dp = di->dp;
	  di->parents[di->nparents].keyp = di->keyp;
	  di->dp = (unsigned char *)di->kv.str;
	  di->keyp = di->data->schemadata + di->data->schemata[di->kv.id];
	  memset(&di->kv, 0, sizeof(di->kv));
	  di->kv.parent = &di->parents[di->nparents].kv;
	  di->nparents++;
	  di->keyname = di->keynames[di->nparents - di->rootlevel];
	  goto di_enterschema;

	case di_leavesub: di_leavesub:
	  if (di->nparents - 1 < di->rootlevel)
	    goto di_bye;
	  di->nparents--;
	  di->dp = di->parents[di->nparents].dp;
	  di->kv = di->parents[di->nparents].kv;
	  di->keyp = di->parents[di->nparents].keyp;
	  di->key = di->data->keys + *di->keyp;
	  di->ddp = (unsigned char *)di->kv.str;
	  di->keyname = di->keynames[di->nparents - di->rootlevel];
	  goto di_nextarrayelement;

        /* special solvable attr handling follows */

	case di_nextsolvablekey: di_nextsolvablekey:
	  if (di->keyname)
	    goto di_nextsolvable;
	  if (di->key->name == RPM_RPMDBID)	/* reached end of list? */
	    goto di_leavesolvablekey;
	  di->key++;
	  /* FALLTHROUGH */

	case di_entersolvablekey: di_entersolvablekey:
	  di->idp = solvabledata_fetch(di->pool->solvables + di->solvid, &di->kv, di->key->name);
	  if (!di->idp || !*di->idp)
	    goto di_nextsolvablekey;
	  if (di->kv.eof)
	    {
	      /* not an array */
	      di->kv.id = *di->idp;
	      di->kv.num = *di->idp;	/* for rpmdbid */
	      di->kv.num2 = 0;		/* for rpmdbid */
	      di->kv.entry = 0;
	      di->state = di_nextsolvablekey;
	      break;
	    }
	  di->kv.entry = -1;
	  /* FALLTHROUGH */

	case di_nextsolvableattr:
	  di->state = di_nextsolvableattr;
	  di->kv.id = *di->idp++;
	  di->kv.entry++;
	  if (!*di->idp)
	    {
	      di->kv.eof = 1;
	      di->state = di_nextsolvablekey;
	    }
	  break;

	}

      /* we have a potential match */
      if (di->matcher.match)
	{
	  const char *str;
	  /* simple pre-check so that we don't need to stringify */
	  if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->matcher.flags & SEARCH_FILES) != 0)
	    if (!datamatcher_checkbasename(&di->matcher, di->kv.str))
	      continue;
	  /* now stringify so that we can do the matching */
	  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
	    {
	      if (di->keyname && (di->key->type == REPOKEY_TYPE_FIXARRAY || di->key->type == REPOKEY_TYPE_FLEXARRAY))
		return 1;
	      continue;
	    }
	  if (!datamatcher_match(&di->matcher, str))
	    continue;
	}
      else
	{
	  /* stringify filelist if requested */
	  if (di->keyname == SOLVABLE_FILELIST && di->key->type == REPOKEY_TYPE_DIRSTRARRAY && (di->flags & SEARCH_FILES) != 0)
	    repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags);
	}
      /* found something! */
      return 1;
    }
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}

struct extdata {
  unsigned char *buf;
  int len;
}


data_skip_schema(Repodata *data, unsigned char *dp, Id schema)
{
  Id *keyp = data->schemadata + data->schemata[schema];
  for (; *keyp; keyp++)
    dp = data_skip_key(data, dp, data->keys + *keyp);
  return dp;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_extend_block(Repodata *data, Id start, Id num)
{
  if (!num)
    return;
  if (!data->incoreoffset)
    {
      /* this also means that data->attrs is NULL */
      data->incoreoffset = solv_calloc_block(num, sizeof(Id), REPODATA_BLOCK);
      data->start = start;
      data->end = start + num;
      return;
    }
  repodata_extend(data, start);
  if (num > 1)
    repodata_extend(data, start + num - 1);
}


repodata_set_filelisttype(Repodata *data, int type)
{
  data->filelisttype = type;
}


repodata_dir2str(Repodata *data, Id did, const char *suf)
{
  Pool *pool = data->repo->pool;
  int l = 0;
  Id parent, comp;
  const char *comps;
  char *p;

  if (!did)
    return suf ? suf : "";
  if (did == 1 && !suf)
    return "/";
  parent = did;
  while (parent)
    {
      comp = dirpool_compid(&data->dirpool, parent);
      comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
      l += strlen(comps);
      parent = dirpool_parent(&data->dirpool, parent);
      if (parent)
	l++;
    }
  if (suf)
    l += strlen(suf) + 1;
  p = pool_alloctmpspace(pool, l + 1) + l;
  *p = 0;
  if (suf)
    {
      p -= strlen(suf);
      strcpy(p, suf);
      *--p = '/';
    }
  parent = did;
  while (parent)
    {
      comp = dirpool_compid(&data->dirpool, parent);
      comps = stringpool_id2str(data->localpool ? &data->spool : &pool->ss, comp);
      l = strlen(comps);
      p -= l;
      strncpy(p, comps, l);
      parent = dirpool_parent(&data->dirpool, parent);
      if (parent)
        *--p = '/';
    }
  return p;
}


dataiterator_skip_repo(Dataiterator *di)
{
  di->nparents = 0;
  di->kv.parent = 0;
  di->rootlevel = 0;
  di->keyname = di->keynames[0];
  di->state = di_nextrepo;
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}


dataiterator_match(Dataiterator *di, Datamatcher *ma)
{
  const char *str;
  if (!(str = repodata_stringify(di->pool, di->data, di->key, &di->kv, di->flags)))
    return 0;
  return ma ? datamatcher_match(ma, str) : 1;
}


find_key_data(Repodata *data, Id solvid, Id keyname, Repokey **keypp)
{
  unsigned char *dp;
  Id schema, *keyp, *kp;
  Repokey *key;

  if (!maybe_load_repodata(data, keyname))
    return 0;
  dp = solvid2data(data, solvid, &schema);
  if (!dp)
    return 0;
  keyp = data->schemadata + data->schemata[schema];
  for (kp = keyp; *kp; kp++)
    if (data->keys[*kp].name == keyname)
      break;
  if (!*kp)
    return 0;
  *keypp = key = data->keys + *kp;
  if (key->type == REPOKEY_TYPE_DELETED)
    return 0;
  if (key->type == REPOKEY_TYPE_VOID || key->type == REPOKEY_TYPE_CONSTANT || key->type == REPOKEY_TYPE_CONSTANTID)
    return dp;	/* no need to forward... */
  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
    return 0;	/* get_data will not work, no need to forward */
  dp = forward_to_key(data, *kp, keyp, dp);
  if (!dp)
    return 0;
  return get_data(data, key, &dp, 0);
}


dataiterator_set_match(Dataiterator *di, const char *match, int flags)
{
  di->flags = (flags & ~SEARCH_THISSOLVID) | (di->flags & SEARCH_THISSOLVID);
  datamatcher_free(&di->matcher);
  memset(&di->matcher, 0, sizeof(di->matcher));
  if (match)
    {
      int error;
      if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
	{
	  di->state = di_bye;
	  return error;
	}
    }
  return 0;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_lookup_kv_uninternalized(Repodata *data, Id solvid, Id keyname, KeyValue *kv)
{
  Id *ap;
  if (!data->attrs || solvid < data->start || solvid >= data->end)
    return 0;
  ap = data->attrs[solvid - data->start];
  if (!ap)
    return 0;
  for (; *ap; ap += 2)
    {
      Repokey *key = data->keys + *ap;
      if (key->name != keyname)
	continue;
      data_fetch_uninternalized(data, key, ap[1], kv);
      return key;
    }
  return 0;
}


repodata_add_stubkey(Repodata *data, Id keyname, Id keytype)
{
  Repokey xkey;

  xkey.name = keyname;
  xkey.type = keytype;
  xkey.storage = KEY_STORAGE_INCORE;
  xkey.size = 0;
  repodata_key2id(data, &xkey, 1);
}


dataiterator_set_search(Dataiterator *di, Repo *repo, Id p)
{
  di->repo = repo;
  di->repoid = 0;
  di->flags &= ~SEARCH_THISSOLVID;
  di->nparents = 0;
  di->rootlevel = 0;
  di->repodataid = 1;
  if (!di->pool->urepos)
    {
      di->state = di_bye;
      return;
    }
  if (!repo)
    {
      di->repoid = 1;
      di->repo = di->pool->repos[di->repoid];
    }
  di->state = di_enterrepo;
  if (p)
    dataiterator_jump_to_solvid(di, p);
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}


repodata_set_id(Repodata *data, Id solvid, Id keyname, Id id)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_ID;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, id);
}


dataiterator_find_keyname(Dataiterator *di, Id keyname)
{
  Id *keyp;
  Repokey *keys = di->data->keys, *key;
  unsigned char *dp;

  for (keyp = di->keyp; *keyp; keyp++)
    if (keys[*keyp].name == keyname)
      break;
  if (!*keyp)
    return 0;
  key = keys + *keyp;
  if (key->type == REPOKEY_TYPE_DELETED)
    return 0;
  if (key->storage != KEY_STORAGE_INCORE && key->storage != KEY_STORAGE_VERTICAL_OFFSET)
    return 0;		/* get_data will not work, no need to forward */
  dp = forward_to_key(di->data, *keyp, di->keyp, di->dp);
  if (!dp)
    return 0;
  di->keyp = keyp;
  return dp;
}


solvid2data(Repodata *data, Id solvid, Id *schemap)
{
  unsigned char *dp = data->incoredata;
  if (!dp)
    return 0;
  if (solvid == SOLVID_META)
    dp += 1;	/* offset of "meta" solvable */
  else if (solvid == SOLVID_POS)
    {
      Pool *pool = data->repo->pool;
      if (data->repo != pool->pos.repo)
	return 0;
      if (data != data->repo->repodata + pool->pos.repodataid)
	return 0;
      dp += pool->pos.dp;
      if (pool->pos.dp != 1)
        {
          *schemap = pool->pos.schema;
          return dp;
	}
    }
  else
    {
      if (solvid < data->start || solvid >= data->end)
	return 0;
      dp += data->incoreoffset[solvid - data->start];
    }
  return data_read_id(dp, schemap);
}


compact_attrdata(Repodata *data, int entry, int nentry)
{
  int i;
  unsigned int attrdatastart = data->attrdatalen;
  unsigned int attriddatastart = data->attriddatalen;
  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
    return;
  for (i = entry; i < nentry; i++)
    {
      Id v, *attrs = data->attrs[i];
      if (!attrs)
	continue;
      for (; *attrs; attrs += 2)
	{
	  switch (data->keys[*attrs].type)
	    {
	    case REPOKEY_TYPE_STR:
	    case REPOKEY_TYPE_BINARY:
	    case_CHKSUM_TYPES:
	      if ((unsigned int)attrs[1] < attrdatastart)
		 attrdatastart = attrs[1];
	      break;
	    case REPOKEY_TYPE_DIRSTRARRAY:
	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
		if ((unsigned int)data->attriddata[v + 1] < attrdatastart)
		  attrdatastart = data->attriddata[v + 1];
	      /* FALLTHROUGH */
	    case REPOKEY_TYPE_IDARRAY:
	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
	      if ((unsigned int)attrs[1] < attriddatastart)
		attriddatastart = attrs[1];
	      break;
	    case REPOKEY_TYPE_FIXARRAY:
	    case REPOKEY_TYPE_FLEXARRAY:
	      return;
	    default:
	      break;
	    }
	}
    }
#if 0
  printf("compact_attrdata %d %d\n", entry, nentry);
  printf("attrdatastart: %d\n", attrdatastart);
  printf("attriddatastart: %d\n", attriddatastart);
#endif
  if (attrdatastart < 1024 * 1024 * 4 && attriddatastart < 1024 * 1024)
    return;
  for (i = entry; i < nentry; i++)
    {
      Id v, *attrs = data->attrs[i];
      if (!attrs)
	continue;
      for (; *attrs; attrs += 2)
	{
	  switch (data->keys[*attrs].type)
	    {
	    case REPOKEY_TYPE_STR:
	    case REPOKEY_TYPE_BINARY:
	    case_CHKSUM_TYPES:
	      attrs[1] -= attrdatastart;
	      break;
	    case REPOKEY_TYPE_DIRSTRARRAY:
	      for (v = attrs[1]; data->attriddata[v] ; v += 2)
		data->attriddata[v + 1] -= attrdatastart;
	      /* FALLTHROUGH */
	    case REPOKEY_TYPE_IDARRAY:
	    case REPOKEY_TYPE_DIRNUMNUMARRAY:
	      attrs[1] -= attriddatastart;
	      break;
	    default:
	      break;
	    }
	}
    }
  if (attrdatastart)
    {
      data->attrdatalen -= attrdatastart;
      memmove(data->attrdata, data->attrdata + attrdatastart, data->attrdatalen);
      data->attrdata = solv_extend_resize(data->attrdata, data->attrdatalen, 1, REPODATA_ATTRDATA_BLOCK);
    }
  if (attriddatastart)
    {
      data->attriddatalen -= attriddatastart;
      memmove(data->attriddata, data->attriddata + attriddatastart, data->attriddatalen * sizeof(Id));
      data->attriddata = solv_extend_resize(data->attriddata, data->attriddatalen, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
    }
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

#define REPODATA_ATTRDATA_BLOCK 1023

#define REPODATA_ATTRIDDATA_BLOCK 63

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_lookup_str(Repodata *data, Id solvid, Id keyname)
{
  unsigned char *dp;
  Repokey *key;
  Id id;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return 0;
  if (key->type == REPOKEY_TYPE_STR)
    return (const char *)dp;
  if (key->type == REPOKEY_TYPE_CONSTANTID)
    id = key->size;
  else if (key->type == REPOKEY_TYPE_ID)
    dp = data_read_id(dp, &id);
  else
    return 0;
  if (data->localpool)
    return stringpool_id2str(&data->spool, id);
  return pool_id2str(data->repo->pool, id);
}


repodata_lookup_void(Repodata *data, Id solvid, Id keyname)
{
  return repodata_lookup_type(data, solvid, keyname) == REPOKEY_TYPE_VOID ? 1 : 0;
}


repodata_lookup_id(Repodata *data, Id solvid, Id keyname)
{
  unsigned char *dp;
  Repokey *key;
  Id id;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return 0;
  if (key->type == REPOKEY_TYPE_CONSTANTID)
    return key->size;
  if (key->type != REPOKEY_TYPE_ID)
    return 0;
  dp = data_read_id(dp, &id);
  return id;
}


repodata_serialize_key(Repodata *data, struct extdata *newincore,
		       struct extdata *newvincore,
		       Id *schema,
		       Repokey *key, Id val)
{
  Id *ida;
  struct extdata *xd;
  unsigned int oldvincorelen = 0;
  Id schemaid, *sp;

  xd = newincore;
  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
    {
      xd = newvincore;
      oldvincorelen = xd->len;
    }
  switch (key->type)
    {
    case REPOKEY_TYPE_VOID:
    case REPOKEY_TYPE_CONSTANT:
    case REPOKEY_TYPE_CONSTANTID:
    case REPOKEY_TYPE_DELETED:
      break;
    case REPOKEY_TYPE_STR:
      data_addblob(xd, data->attrdata + val, strlen((char *)(data->attrdata + val)) + 1);
      break;
    case REPOKEY_TYPE_MD5:
      data_addblob(xd, data->attrdata + val, SIZEOF_MD5);
      break;
    case REPOKEY_TYPE_SHA1:
      data_addblob(xd, data->attrdata + val, SIZEOF_SHA1);
      break;
    case REPOKEY_TYPE_SHA224:
      data_addblob(xd, data->attrdata + val, SIZEOF_SHA224);
      break;
    case REPOKEY_TYPE_SHA256:
      data_addblob(xd, data->attrdata + val, SIZEOF_SHA256);
      break;
    case REPOKEY_TYPE_SHA384:
      data_addblob(xd, data->attrdata + val, SIZEOF_SHA384);
      break;
    case REPOKEY_TYPE_SHA512:
      data_addblob(xd, data->attrdata + val, SIZEOF_SHA512);
      break;
    case REPOKEY_TYPE_NUM:
      if (val & 0x80000000)
	{
	  data_addid64(xd, data->attrnum64data[val ^ 0x80000000]);
	  break;
	}
      /* FALLTHROUGH */
    case REPOKEY_TYPE_ID:
    case REPOKEY_TYPE_DIR:
      data_addid(xd, val);
      break;
    case REPOKEY_TYPE_BINARY:
      {
	Id len;
	unsigned char *dp = data_read_id(data->attrdata + val, &len);
	dp += (unsigned int)len;
	data_addblob(xd, data->attrdata + val, dp - (data->attrdata + val));
      }
      break;
    case REPOKEY_TYPE_IDARRAY:
      for (ida = data->attriddata + val; *ida; ida++)
	data_addideof(xd, ida[0], ida[1] ? 0 : 1);
      break;
    case REPOKEY_TYPE_DIRNUMNUMARRAY:
      for (ida = data->attriddata + val; *ida; ida += 3)
	{
	  data_addid(xd, ida[0]);
	  data_addid(xd, ida[1]);
	  data_addideof(xd, ida[2], ida[3] ? 0 : 1);
	}
      break;
    case REPOKEY_TYPE_DIRSTRARRAY:
      for (ida = data->attriddata + val; *ida; ida += 2)
	{
	  data_addideof(xd, ida[0], ida[2] ? 0 : 1);
	  data_addblob(xd, data->attrdata + ida[1], strlen((char *)(data->attrdata + ida[1])) + 1);
	}
      break;
    case REPOKEY_TYPE_FIXARRAY:
      {
	int num = 0;
	schemaid = 0;
	for (ida = data->attriddata + val; *ida; ida++)
	  {
	    Id *kp;
	    sp = schema;
	    kp = data->xattrs[-*ida];
	    if (!kp)
	      continue;		/* ignore empty elements */
	    num++;
	    for (; *kp; kp += 2)
	      *sp++ = *kp;
	    *sp = 0;
	    if (!schemaid)
	      schemaid = repodata_schema2id(data, schema, 1);
	    else if (schemaid != repodata_schema2id(data, schema, 0))
	      {
	 	pool_debug(data->repo->pool, SOLV_ERROR, "repodata_serialize_key: fixarray substructs with different schemas\n");
		num = 0;
		break;
	      }
	  }
	data_addid(xd, num);
	if (!num)
	  break;
	data_addid(xd, schemaid);
	for (ida = data->attriddata + val; *ida; ida++)
	  {
	    Id *kp = data->xattrs[-*ida];
	    if (!kp)
	      continue;
	    for (; *kp; kp += 2)
	      repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
	  }
	break;
      }
    case REPOKEY_TYPE_FLEXARRAY:
      {
	int num = 0;
	for (ida = data->attriddata + val; *ida; ida++)
	  num++;
	data_addid(xd, num);
	for (ida = data->attriddata + val; *ida; ida++)
	  {
	    Id *kp = data->xattrs[-*ida];
	    if (!kp)
	      {
	        data_addid(xd, 0);	/* XXX */
	        continue;
	      }
	    sp = schema;
	    for (;*kp; kp += 2)
	      *sp++ = *kp;
	    *sp = 0;
	    schemaid = repodata_schema2id(data, schema, 1);
	    data_addid(xd, schemaid);
	    kp = data->xattrs[-*ida];
	    for (;*kp; kp += 2)
	      repodata_serialize_key(data, newincore, newvincore, schema, data->keys + *kp, kp[1]);
	  }
	break;
      }
    default:
      pool_debug(data->repo->pool, SOLV_FATAL, "repodata_serialize_key: don't know how to handle type %d\n", key->type);
      exit(1);
    }
  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
    {
      /* put offset/len in incore */
      data_addid(newincore, data->lastverticaloffset + oldvincorelen);
      oldvincorelen = xd->len - oldvincorelen;
      data_addid(newincore, oldvincorelen);
    }
}


forward_to_key(Repodata *data, Id keyid, Id *keyp, unsigned char *dp)
{
  Id k;

  if (!keyid)
    return 0;
  if (data->mainschemaoffsets && dp == data->incoredata + data->mainschemaoffsets[0] && keyp == data->schemadata + data->schemata[data->mainschema])
    {
      int i;
      for (i = 0; (k = *keyp++) != 0; i++)
        if (k == keyid)
	  return data->incoredata + data->mainschemaoffsets[i];
      return 0;
    }
  while ((k = *keyp++) != 0)
    {
      if (k == keyid)
	return dp;
      if (data->keys[k].storage == KEY_STORAGE_VERTICAL_OFFSET)
	{
	  dp = data_skip(dp, REPOKEY_TYPE_ID);	/* skip offset */
	  dp = data_skip(dp, REPOKEY_TYPE_ID);	/* skip length */
	  continue;
	}
      if (data->keys[k].storage != KEY_STORAGE_INCORE)
	continue;
      dp = data_skip_key(data, dp, data->keys + k);
    }
  return 0;
}


dataiterator_entersub(Dataiterator *di)
{
  if (di->state == di_nextarrayelement)
    di->state = di_entersub;
}


repodata_add_array(Repodata *data, Id handle, Id keyname, Id keytype, int entrysize)
{
  int oldsize;
  Id *ida, *pp, **ppp;

  /* check if it is the same as last time, this speeds things up a lot */
  if (handle == data->lasthandle && data->keys[data->lastkey].name == keyname && data->keys[data->lastkey].type == keytype && data->attriddatalen == data->lastdatalen)
    {
      /* great! just append the new data */
      data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
      data->attriddatalen--;	/* overwrite terminating 0  */
      data->lastdatalen += entrysize;
      return;
    }

  ppp = repodata_get_attrp(data, handle);
  pp = *ppp;
  if (pp)
    {
      for (; *pp; pp += 2)
        if (data->keys[*pp].name == keyname)
          break;
    }
  if (!pp || !*pp || data->keys[*pp].type != keytype)
    {
      /* not found. allocate new key */
      Repokey key;
      Id keyid;
      key.name = keyname;
      key.type = keytype;
      key.size = 0;
      key.storage = KEY_STORAGE_INCORE;
      data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
      keyid = repodata_key2id(data, &key, 1);
      repodata_insert_keyid(data, handle, keyid, data->attriddatalen, 1);
      data->lasthandle = handle;
      data->lastkey = keyid;
      data->lastdatalen = data->attriddatalen + entrysize + 1;
      return;
    }
  oldsize = 0;
  for (ida = data->attriddata + pp[1]; *ida; ida += entrysize)
    oldsize += entrysize;
  if (ida + 1 == data->attriddata + data->attriddatalen)
    {
      /* this was the last entry, just append it */
      data->attriddata = solv_extend(data->attriddata, data->attriddatalen, entrysize, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
      data->attriddatalen--;	/* overwrite terminating 0  */
    }
  else
    {
      /* too bad. move to back. */
      data->attriddata = solv_extend(data->attriddata, data->attriddatalen,  oldsize + entrysize + 1, sizeof(Id), REPODATA_ATTRIDDATA_BLOCK);
      memcpy(data->attriddata + data->attriddatalen, data->attriddata + pp[1], oldsize * sizeof(Id));
      pp[1] = data->attriddatalen;
      data->attriddatalen += oldsize;
    }
  data->lasthandle = handle;
  data->lastkey = *pp;
  data->lastdatalen = data->attriddatalen + entrysize + 1;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

#define REPODATA_ATTRIDDATA_BLOCK 63

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_create_stubs(Repodata *data)
{
  Repo *repo = data->repo;
  Pool *pool = repo->pool;
  Repodata *sdata;
  int *stubdataids;
  Dataiterator di;
  Id xkeyname = 0;
  int i, cnt = 0;

  dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
  while (dataiterator_step(&di))
    if (di.data == data)
      cnt++;
  dataiterator_free(&di);
  if (!cnt)
    return data;
  stubdataids = solv_calloc(cnt, sizeof(*stubdataids));
  for (i = 0; i < cnt; i++)
    {
      sdata = repodata_add_stub(&data);
      stubdataids[i] = sdata - repo->repodata;
    }
  i = 0;
  dataiterator_init(&di, pool, repo, SOLVID_META, REPOSITORY_EXTERNAL, 0, 0);
  sdata = 0;
  while (dataiterator_step(&di))
    {
      if (di.data != data)
	continue;
      if (di.key->name == REPOSITORY_EXTERNAL && !di.nparents)
	{
	  dataiterator_entersub(&di);
	  sdata = repo->repodata + stubdataids[i++];
	  xkeyname = 0;
	  continue;
	}
      repodata_set_kv(sdata, SOLVID_META, di.key->name, di.key->type, &di.kv);
      if (di.key->name == REPOSITORY_KEYS && di.key->type == REPOKEY_TYPE_IDARRAY)
	{
	  if (!xkeyname)
	    {
	      if (!di.kv.eof)
		xkeyname = di.kv.id;
	    }
	  else
	    {
	      repodata_add_stubkey(sdata, xkeyname, di.kv.id);
	      if (xkeyname == SOLVABLE_FILELIST)
	        repodata_set_filelisttype(sdata, REPODATA_FILELIST_EXTENSION);
	      xkeyname = 0;
	    }
	}
    }
  dataiterator_free(&di);
  for (i = 0; i < cnt; i++)
    repodata_internalize(repo->repodata + stubdataids[i]);
  solv_free(stubdataids);
  return data;
}


repodata_add_poolstr_array(Repodata *data, Id solvid, Id keyname,
			   const char *str)
{
  Id id;
  if (data->localpool)
    id = stringpool_str2id(&data->spool, str, 1);
  else
    id = pool_str2id(data->repo->pool, str, 1);
  repodata_add_idarray(data, solvid, keyname, id);
}


repodata_empty(Repodata *data, int localpool)
{
  void (*loadcallback)(Repodata *) = data->loadcallback;
  int state = data->state;
  repodata_freedata(data);
  repodata_initdata(data, data->repo, localpool);
  data->state = state;
  data->loadcallback = loadcallback;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_unset_uninternalized(Repodata *data, Id solvid, Id keyname)
{
  Id *pp, *ap, **app;
  app = repodata_get_attrp(data, solvid);
  ap = *app;
  if (!ap)
    return;
  if (!keyname)
    {
      *app = 0;		/* delete all attributes */
      return;
    }
  for (; *ap; ap += 2)
    if (data->keys[*ap].name == keyname)
      break;
  if (!*ap)
    return;
  pp = ap;
  ap += 2;
  for (; *ap; ap += 2)
    {
      if (data->keys[*ap].name == keyname)
	continue;
      *pp++ = ap[0];
      *pp++ = ap[1];
    }
  *pp = 0;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}


repodata_lookup_packed_dirstrarray(Repodata *data, Id solvid, Id keyname)
{
  static unsigned char wrongtype[2] = { 0x00 /* dir id 0 */, 0 /* "" */ };
  unsigned char *dp;
  Id schema, *keyp, *kp;
  Repokey *key;

  if (!data->incoredata || !data->incoreoffset[solvid - data->start])
    return 0;
  dp = data->incoredata + data->incoreoffset[solvid - data->start];
  dp = data_read_id(dp, &schema);
  keyp = data->schemadata + data->schemata[schema];
  for (kp = keyp; *kp; kp++)
    if (data->keys[*kp].name == keyname)
      break;
  if (!*kp)
    return 0;
  key = data->keys + *kp;
  if (key->type != REPOKEY_TYPE_DIRSTRARRAY)
    return wrongtype;
  dp = forward_to_key(data, *kp, keyp, dp);
  if (key->storage == KEY_STORAGE_INCORE)
    return dp;
  if (key->storage == KEY_STORAGE_VERTICAL_OFFSET && dp)
    {
      Id off, len;
      dp = data_read_id(dp, &off);
      data_read_id(dp, &len);
      return get_vertical_data(data, key, off, len);
    }
  return 0;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_schema2id(Repodata *data, Id *schema, int create)
{
  int h, len, i;
  Id *sp, cid;
  Id *schematahash;

  if (!*schema)
    return 0;	/* XXX: allow empty schema? */
  if ((schematahash = data->schematahash) == 0)
    {
      data->schematahash = schematahash = solv_calloc(256, sizeof(Id));
      for (i = 1; i < data->nschemata; i++)
	{
	  for (sp = data->schemadata + data->schemata[i], h = 0; *sp;)
	    h = h * 7 + *sp++;
	  h &= 255;
	  schematahash[h] = i;
	}
      data->schemadata = solv_extend_resize(data->schemadata, data->schemadatalen, sizeof(Id), SCHEMATADATA_BLOCK);
      data->schemata = solv_extend_resize(data->schemata, data->nschemata, sizeof(Id), SCHEMATA_BLOCK);
    }

  for (sp = schema, len = 0, h = 0; *sp; len++)
    h = h * 7 + *sp++;
  h &= 255;
  len++;

  cid = schematahash[h];
  if (cid)
    {
      if ((data->schemata[cid] + len <= data->schemadatalen) &&
			  !memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
        return cid;
      /* cache conflict, do a slow search */
      for (cid = 1; cid < data->nschemata; cid++)
        if ((data->schemata[cid] + len <= data->schemadatalen) &&
				!memcmp(data->schemadata + data->schemata[cid], schema, len * sizeof(Id)))
          return cid;
    }
  /* a new one */
  if (!create)
    return 0;
  data->schemadata = solv_extend(data->schemadata, data->schemadatalen, len, sizeof(Id), SCHEMATADATA_BLOCK);
  data->schemata = solv_extend(data->schemata, data->nschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
  /* add schema */
  memcpy(data->schemadata + data->schemadatalen, schema, len * sizeof(Id));
  data->schemata[data->nschemata] = data->schemadatalen;
  data->schemadatalen += len;
  schematahash[h] = data->nschemata;
#if 0
fprintf(stderr, "schema2id: new schema\n");
#endif
  return data->nschemata++;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

#define SCHEMATA_BLOCK 31

#define SCHEMATADATA_BLOCK 255

struct extdata {
  unsigned char *buf;
  int len;
}


get_data(Repodata *data, Repokey *key, unsigned char **dpp, int advance)
{
  unsigned char *dp = *dpp;

  if (!dp)
    return 0;
  if (key->storage == KEY_STORAGE_INCORE)
    {
      if (advance)
        *dpp = data_skip_key(data, dp, key);
      return dp;
    }
  else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
    {
      Id off, len;
      dp = data_read_id(dp, &off);
      dp = data_read_id(dp, &len);
      if (advance)
        *dpp = dp;
      return get_vertical_data(data, key, off, len);
    }
  return 0;
}


repodata_set_constantid(Repodata *data, Id solvid, Id keyname, Id id)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_CONSTANTID;
  key.size = id;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


dataiterator_prepend_keyname(Dataiterator *di, Id keyname)
{
  int i;

  if (di->nkeynames >= sizeof(di->keynames)/sizeof(*di->keynames) - 2)
    {
      di->state = di_bye;	/* sorry */
      return;
    }
  for (i = di->nkeynames + 1; i > 0; i--)
    di->keynames[i] = di->keynames[i - 1];
  di->keynames[0] = di->keyname = keyname;
  di->nkeynames++;
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_memused(Repodata *data)
{
  return data->incoredatalen + data->vincorelen;
}


evrid2vrstr(Pool *pool, Id evrid)
{
  const char *p, *evr = pool_id2str(pool, evrid);
  if (!evr)
    return evr;
  for (p = evr; *p >= '0' && *p <= '9'; p++)
    ;
  return p != evr && *p == ':' && p[1] ? p + 1 : evr;
}


dataiterator_jump_to_repo(Dataiterator *di, Repo *repo)
{
  di->nparents = 0;
  di->kv.parent = 0;
  di->rootlevel = 0;
  di->repo = repo;
  di->repoid = 0;	/* 0 means stay at repo */
  di->repodataid = 1;
  di->solvid = 0;
  di->flags &= ~SEARCH_THISSOLVID;
  di->state = di_enterrepo;
}


repodata_add_dirnumnum(Repodata *data, Id solvid, Id keyname, Id dir, Id num, Id num2)
{
  assert(dir);
#if 0
fprintf(stderr, "repodata_add_dirnumnum %d %d %d %d (%d)\n", solvid, dir, num, num2, data->attriddatalen);
#endif
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_DIRNUMNUMARRAY, 3);
  data->attriddata[data->attriddatalen++] = dir;
  data->attriddata[data->attriddatalen++] = num;
  data->attriddata[data->attriddatalen++] = num2;
  data->attriddata[data->attriddatalen++] = 0;
}


dataiterator_init(Dataiterator *di, Pool *pool, Repo *repo, Id p, Id keyname, const char *match, int flags)
{
  memset(di, 0, sizeof(*di));
  di->pool = pool;
  di->flags = flags & ~SEARCH_THISSOLVID;
  if (!pool || (repo && repo->pool != pool))
    {
      di->state = di_bye;
      return -1;
    }
  if (match)
    {
      int error;
      if ((error = datamatcher_init(&di->matcher, match, flags)) != 0)
	{
	  di->state = di_bye;
	  return error;
	}
    }
  di->keyname = keyname;
  di->keynames[0] = keyname;
  dataiterator_set_search(di, repo, p);
  return 0;
}


repodata_fill_keyskip(Repodata *data, Id solvid, Id *keyskip)
{
  const Id *keyp;
  Id maxkeyname, value;
  keyp = repodata_lookup_schemakeys(data, solvid);
  if (!keyp)
    return keyskip;	/* no keys for this solvid */
  if (!keyskip)
    keyskip = alloc_keyskip();
  maxkeyname = keyskip[0];
  value = keyskip[1] + data->repodataid;
  for (; *keyp; keyp++)
    {
      Id keyname = data->keys[*keyp].name;
      if (keyname >= maxkeyname)
	{
	  int newmax = (keyname | 255) + 1;
	  keyskip = solv_realloc2(keyskip, 3 + newmax, sizeof(Id));
	  memset(keyskip + (3 + maxkeyname), 0, (newmax - maxkeyname) * sizeof(Id));
	  keyskip[0] = maxkeyname = newmax;
	}
      keyskip[3 + keyname] = value;
    }
  return keyskip;
}


repodata_add_flexarray(Repodata *data, Id solvid, Id keyname, Id handle)
{
  repodata_add_array(data, solvid, keyname, REPOKEY_TYPE_FLEXARRAY, 1);
  data->attriddata[data->attriddatalen++] = handle;
  data->attriddata[data->attriddatalen++] = 0;
}


maybe_load_repodata(Repodata *data, Id keyname)
{
  if (keyname && !repodata_precheck_keyname(data, keyname))
    return 0;	/* do not bother... */
  if (data->state == REPODATA_AVAILABLE || data->state == REPODATA_LOADING)
    return 1;
  if (data->state == REPODATA_ERROR)
    return 0;
  return maybe_load_repodata_stub(data, keyname);
}


repodata_merge_attrs(Repodata *data, Id dest, Id src)
{
  Id *keyp;
  if (dest == src || !data->attrs || !(keyp = data->attrs[src - data->start]))
    return;
  for (; *keyp; keyp += 2)
    repodata_insert_keyid(data, dest, keyp[0], keyp[1], 0);
}


repodata_translate_id(Repodata *data, Repodata *fromdata, Id id, int create)
{
  const char *s;
  if (!id || !data || !fromdata)
    return id;
  if (data == fromdata || (!data->localpool && !fromdata->localpool))
    return id;
  if (fromdata->localpool)
    s = stringpool_id2str(&fromdata->spool, id);
  else
    s = pool_id2str(data->repo->pool, id);
  if (data->localpool)
    return stringpool_str2id(&data->spool, s, create);
  else
    return pool_str2id(data->repo->pool, s, create);
}


repodata_search_uninternalized(Repodata *data, Id solvid, Id keyname, int flags, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  Id *ap;
  int stop;
  Solvable *s;
  KeyValue kv;

  if (!data->attrs || solvid < data->start || solvid >= data->end)
    return;
  ap = data->attrs[solvid - data->start];
  if (!ap)
    return;
  for (; *ap; ap += 2)
    {
      Repokey *key = data->keys + *ap;
      if (keyname && key->name != keyname)
	continue;
      s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
      kv.entry = 0;
      do
	{
	  data_fetch_uninternalized(data, key, ap[1], &kv);
	  stop = callback(cbdata, s, data, key, &kv);
	  kv.entry++;
	}
      while (!kv.eof && !stop);
      if (keyname || stop > SEARCH_NEXT_KEY)
	return;
    }
}


repodata_set_checksum(Repodata *data, Id solvid, Id keyname, Id type,
		      const char *str)
{
  unsigned char buf[64];
  int l;

  if (!(l = solv_chksum_len(type)))
    return;
  if (l > sizeof(buf) || solv_hex2bin(&str, buf, l) != l)
    return;
  repodata_set_bin_checksum(data, solvid, keyname, type, buf);
}


repodata_set_constant(Repodata *data, Id solvid, Id keyname, unsigned int constant)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_CONSTANT;
  key.size = constant;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


repodata_set_kv(Repodata *data, Id solvid, Id keyname, Id keytype, KeyValue *kv)
{
  switch (keytype)
    {
    case REPOKEY_TYPE_ID:
      repodata_set_id(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_CONSTANTID:
      repodata_set_constantid(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_IDARRAY:
      repodata_add_idarray(data, solvid, keyname, kv->id);
      break;
    case REPOKEY_TYPE_STR:
      repodata_set_str(data, solvid, keyname, kv->str);
      break;
    case REPOKEY_TYPE_VOID:
      repodata_set_void(data, solvid, keyname);
      break;
    case REPOKEY_TYPE_NUM:
      repodata_set_num(data, solvid, keyname, SOLV_KV_NUM64(kv));
      break;
    case REPOKEY_TYPE_CONSTANT:
      repodata_set_constant(data, solvid, keyname, kv->num);
      break;
    case REPOKEY_TYPE_DIRNUMNUMARRAY:
      if (kv->id)
        repodata_add_dirnumnum(data, solvid, keyname, kv->id, kv->num, kv->num2);
      break;
    case REPOKEY_TYPE_DIRSTRARRAY:
      repodata_add_dirstr(data, solvid, keyname, kv->id, kv->str);
      break;
    case_CHKSUM_TYPES:
      repodata_set_bin_checksum(data, solvid, keyname, keytype, (const unsigned char *)kv->str);
      break;
    default:
      break;
    }
}


get_vertical_data(Repodata *data, Repokey *key, Id off, Id len)
{
  unsigned char *dp;
  if (len <= 0)
    return 0;
  if (off >= data->lastverticaloffset)
    {
      off -= data->lastverticaloffset;
      if ((unsigned int)off + len > data->vincorelen)
	return 0;
      return data->vincore + off;
    }
  if ((unsigned int)off + len > key->size)
    return 0;
  /* we now have the offset, go into vertical */
  off += data->verticaloffset[key - data->keys];
  /* fprintf(stderr, "key %d page %d\n", key->name, off / REPOPAGE_BLOBSIZE); */
  dp = repopagestore_load_page_range(&data->store, off / REPOPAGE_BLOBSIZE, (off + len - 1) / REPOPAGE_BLOBSIZE);
  data->storestate++;
  if (dp)
    dp += off % REPOPAGE_BLOBSIZE;
  return dp;
}


static inline const char *strchrnul(const char *str, char x)
{
  const char *p = strchr(str, x);
  return p ? p : str + strlen(str);
}


solvabledata_fetch(Solvable *s, KeyValue *kv, Id keyname)
{
  kv->id = keyname;
  switch (keyname)
    {
    case SOLVABLE_NAME:
      kv->eof = 1;
      return &s->name;
    case SOLVABLE_ARCH:
      kv->eof = 1;
      return &s->arch;
    case SOLVABLE_EVR:
      kv->eof = 1;
      return &s->evr;
    case SOLVABLE_VENDOR:
      kv->eof = 1;
      return &s->vendor;
    case SOLVABLE_PROVIDES:
      kv->eof = 0;
      return s->provides ? s->repo->idarraydata + s->provides : 0;
    case SOLVABLE_OBSOLETES:
      kv->eof = 0;
      return s->obsoletes ? s->repo->idarraydata + s->obsoletes : 0;
    case SOLVABLE_CONFLICTS:
      kv->eof = 0;
      return s->conflicts ? s->repo->idarraydata + s->conflicts : 0;
    case SOLVABLE_REQUIRES:
      kv->eof = 0;
      return s->requires ? s->repo->idarraydata + s->requires : 0;
    case SOLVABLE_RECOMMENDS:
      kv->eof = 0;
      return s->recommends ? s->repo->idarraydata + s->recommends : 0;
    case SOLVABLE_SUPPLEMENTS:
      kv->eof = 0;
      return s->supplements ? s->repo->idarraydata + s->supplements : 0;
    case SOLVABLE_SUGGESTS:
      kv->eof = 0;
      return s->suggests ? s->repo->idarraydata + s->suggests : 0;
    case SOLVABLE_ENHANCES:
      kv->eof = 0;
      return s->enhances ? s->repo->idarraydata + s->enhances : 0;
    case RPM_RPMDBID:
      kv->eof = 1;
      return s->repo->rpmdbid ? s->repo->rpmdbid + (s - s->repo->pool->solvables - s->repo->start) : 0;
    default:
      return 0;
    }
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}


dataiterator_set_keyname(Dataiterator *di, Id keyname)
{
  di->nkeynames = 0;
  di->keyname = keyname;
  di->keynames[0] = keyname;
}

#define _GNU_SOURCE

#define REPODATA_BLOCK 255

#define SCHEMATA_BLOCK 31

#define SCHEMATADATA_BLOCK 255

#define REPODATA_ATTRS_BLOCK 31

#define EXTDATA_BLOCK 1023

#define REPODATA_ATTRIDDATA_BLOCK 63

#define REPODATA_ATTRDATA_BLOCK 1023

#define REPODATA_ATTRNUM64DATA_BLOCK 15

#define DIRCACHE_SIZE 41	/* < 1k */

struct extdata {
  unsigned char *buf;
  int len;
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

enum {
  di_bye,

  di_enterrepo,
  di_entersolvable,
  di_enterrepodata,
  di_enterschema,
  di_enterkey,

  di_nextattr,
  di_nextkey,
  di_nextrepodata,
  di_nextsolvable,
  di_nextrepo,

  di_enterarray,
  di_nextarrayelement,

  di_entersub,
  di_leavesub,

  di_nextsolvablekey,
  di_entersolvablekey,
  di_nextsolvableattr
}


data_addid64(struct extdata *xd, unsigned long long x)
{
  if (x >= 0x100000000)
    {
      if ((x >> 35) != 0)
	{
	  data_addid(xd, (Id)(x >> 35));
	  xd->buf[xd->len - 1] |= 128;
	}
      data_addid(xd, (Id)((unsigned int)x | 0x80000000));
      xd->buf[xd->len - 5] = (x >> 28) | 128;
    }
  else
    data_addid(xd, (Id)x);
}


repodata_disable_paging(Repodata *data)
{
  if (maybe_load_repodata(data, 0))
    {
      repopagestore_disable_paging(&data->store);
      data->storestate++;
    }
}


repodata_search_array(Repodata *data, Id solvid, Id keyname, int flags, Repokey *key, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  Solvable *s = solvid > 0 ? data->repo->pool->solvables + solvid : 0;
  unsigned char *dp = (unsigned char *)kv->str;
  int stop;
  Id schema = 0;

  if (!dp || kv->entry != -1)
    return 0;
  while (++kv->entry < (int)kv->num)
    {
      if (kv->entry)
	dp = data_skip_schema(data, dp, schema);
      if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
	dp = data_read_id(dp, &schema);
      kv->id = schema;
      kv->str = (const char *)dp;
      kv->eof = kv->entry == kv->num - 1 ? 1 : 0;
      stop = callback(cbdata, s, data, key, kv);
      if (stop && stop != SEARCH_ENTERSUB)
	return stop;
      if ((flags & SEARCH_SUB) != 0 || stop == SEARCH_ENTERSUB)
        repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
    }
  if ((flags & SEARCH_ARRAYSENTINEL) != 0)
    {
      if (kv->entry)
	dp = data_skip_schema(data, dp, schema);
      kv->id = 0;
      kv->str = (const char *)dp;
      kv->eof = 2;
      return callback(cbdata, s, data, key, kv);
    }
  return 0;
}


repodata_set_sourcepkg(Repodata *data, Id solvid, const char *sourcepkg)
{
  Pool *pool = data->repo->pool;
  Solvable *s = pool->solvables + solvid;
  const char *p, *sevr, *sarch, *name, *evr;

  p = strrchr(sourcepkg, '.');
  if (!p || strcmp(p, ".rpm") != 0)
    {
      if (*sourcepkg)
        repodata_set_str(data, solvid, SOLVABLE_SOURCENAME, sourcepkg);
      return;
    }
  p--;
  while (p > sourcepkg && *p != '.')
    p--;
  if (*p != '.' || p == sourcepkg)
    return;
  sarch = p-- + 1;
  while (p > sourcepkg && *p != '-')
    p--;
  if (*p != '-' || p == sourcepkg)
    return;
  p--;
  while (p > sourcepkg && *p != '-')
    p--;
  if (*p != '-' || p == sourcepkg)
    return;
  sevr = p + 1;
  pool = s->repo->pool;

  name = pool_id2str(pool, s->name);
  if (name && !strncmp(sourcepkg, name, sevr - sourcepkg - 1) && name[sevr - sourcepkg - 1] == 0)
    repodata_set_void(data, solvid, SOLVABLE_SOURCENAME);
  else
    repodata_set_id(data, solvid, SOLVABLE_SOURCENAME, pool_strn2id(pool, sourcepkg, sevr - sourcepkg - 1, 1));

  evr = evrid2vrstr(pool, s->evr);
  if (evr && !strncmp(sevr, evr, sarch - sevr - 1) && evr[sarch - sevr - 1] == 0)
    repodata_set_void(data, solvid, SOLVABLE_SOURCEEVR);
  else
    repodata_set_id(data, solvid, SOLVABLE_SOURCEEVR, pool_strn2id(pool, sevr, sarch - sevr - 1, 1));

  if (!strcmp(sarch, "src.rpm"))
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_SRC);
  else if (!strcmp(sarch, "nosrc.rpm"))
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, ARCH_NOSRC);
  else
    repodata_set_constantid(data, solvid, SOLVABLE_SOURCEARCH, pool_strn2id(pool, sarch, strlen(sarch) - 4, 1));
}


repodata_lookup_schemakeys(Repodata *data, Id solvid)
{
  Id schema;
  if (!maybe_load_repodata(data, 0))
    return 0;
  if (!solvid2data(data, solvid, &schema))
    return 0;
  return data->schemadata + data->schemata[schema];
}


repodata_lookup_bin_checksum(Repodata *data, Id solvid, Id keyname, Id *typep)
{
  unsigned char *dp;
  Repokey *key;

  dp = find_key_data(data, solvid, keyname, &key);
  if (!dp)
    return 0;
  switch (key->type)
    {
    case_CHKSUM_TYPES:
      break;
    default:
      return 0;
    }
  *typep = key->type;
  return dp;
}


repodata_translate_dir_slow(Repodata *data, Repodata *fromdata, Id dir, int create, Id *cache)
{
  Id parent, compid;
  if (!dir)
    {
      /* make sure that the dirpool has an entry */
      if (create && !data->dirpool.ndirs)
        dirpool_add_dir(&data->dirpool, 0, 0, create);
      return 0;
    }
  parent = dirpool_parent(&fromdata->dirpool, dir);
  if (parent)
    {
      if (!(parent = repodata_translate_dir(data, fromdata, parent, create, cache)))
	return 0;
    }
  compid = dirpool_compid(&fromdata->dirpool, dir);
  if (compid > 1 && (data->localpool || fromdata->localpool))
    {
      if (!(compid = repodata_translate_id(data, fromdata, compid, create)))
	return 0;
    }
  if (!(compid = dirpool_add_dir(&data->dirpool, parent, compid, create)))
    return 0;
  if (cache)
    {
      cache[(dir & 255) * 2] = dir;
      cache[(dir & 255) * 2 + 1] = compid;
    }
  return compid;
}


repodata_stringify(Pool *pool, Repodata *data, Repokey *key, KeyValue *kv, int flags)
{
  switch (key->type)
    {
    case REPOKEY_TYPE_ID:
    case REPOKEY_TYPE_CONSTANTID:
    case REPOKEY_TYPE_IDARRAY:
      if (data && data->localpool)
	kv->str = stringpool_id2str(&data->spool, kv->id);
      else
	kv->str = pool_id2str(pool, kv->id);
      if ((flags & SEARCH_SKIP_KIND) != 0 && key->storage == KEY_STORAGE_SOLVABLE && (key->name == SOLVABLE_NAME || key->type == REPOKEY_TYPE_IDARRAY))
	{
	  const char *s;
	  for (s = kv->str; *s >= 'a' && *s <= 'z'; s++)
	    ;
	  if (*s == ':' && s > kv->str)
	    kv->str = s + 1;
	}
      return kv->str;
    case REPOKEY_TYPE_STR:
      return kv->str;
    case REPOKEY_TYPE_DIRSTRARRAY:
      if (!(flags & SEARCH_FILES))
	return kv->str;	/* match just the basename */
      if (kv->num)
	return kv->str;	/* already stringified */
      /* Put the full filename into kv->str.  */
      kv->str = repodata_dir2str(data, kv->id, kv->str);
      kv->num = 1;	/* mark stringification */
      return kv->str;
    case_CHKSUM_TYPES:
      if (!(flags & SEARCH_CHECKSUMS))
	return 0;	/* skip em */
      if (kv->num)
	return kv->str;	/* already stringified */
      kv->str = repodata_chk2str(data, key->type, (const unsigned char *)kv->str);
      kv->num = 1;	/* mark stringification */
      return kv->str;
    default:
      return 0;
    }
}


datamatcher_checkbasename(Datamatcher *ma, const char *basename)
{
  int l;
  const char *match = ma->matchdata;
  if (!match)
    return 1;
  switch (ma->flags & SEARCH_STRINGMASK)
    {
    case SEARCH_STRING:
      break;
    case SEARCH_STRINGEND:
      if (match != ma->match)
	break;		/* had slash, do exact match on basename */
      /* FALLTHROUGH */
    case SEARCH_GLOB:
      /* check if the basename ends with match */
      l = strlen(basename) - strlen(match);
      if (l < 0)
	return 0;
      basename += l;
      break;
    default:
      return 1;	/* maybe matches */
    }
  if ((ma->flags & SEARCH_NOCASE) != 0)
    return !strcasecmp(match, basename);
  else
    return !strcmp(match, basename);
}


repodata_extend(Repodata *data, Id p)
{
  if (data->start == data->end)
    data->start = data->end = p;
  if (p >= data->end)
    {
      int old = data->end - data->start;
      int new = p - data->end + 1;
      if (data->attrs)
	{
	  data->attrs = solv_extend(data->attrs, old, new, sizeof(Id *), REPODATA_BLOCK);
	  memset(data->attrs + old, 0, new * sizeof(Id *));
	}
      data->incoreoffset = solv_extend(data->incoreoffset, old, new, sizeof(Id), REPODATA_BLOCK);
      memset(data->incoreoffset + old, 0, new * sizeof(Id));
      data->end = p + 1;
    }
  if (p < data->start)
    {
      int old = data->end - data->start;
      int new = data->start - p;
      if (data->attrs)
	{
	  data->attrs = solv_extend_resize(data->attrs, old + new, sizeof(Id *), REPODATA_BLOCK);
	  memmove(data->attrs + new, data->attrs, old * sizeof(Id *));
	  memset(data->attrs, 0, new * sizeof(Id *));
	}
      data->incoreoffset = solv_extend_resize(data->incoreoffset, old + new, sizeof(Id), REPODATA_BLOCK);
      memmove(data->incoreoffset + new, data->incoreoffset, old * sizeof(Id));
      memset(data->incoreoffset, 0, new * sizeof(Id));
      data->start = p;
    }
}

#define REPODATA_BLOCK 255

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_chk2str(Repodata *data, Id type, const unsigned char *buf)
{
  int l;

  if (!(l = solv_chksum_len(type)))
    return "";
  return pool_bin2hex(data->repo->pool, buf, l);
}


dataiterator_init_clone(Dataiterator *di, Dataiterator *from)
{
  *di = *from;
  if (di->dupstr)
    {
      if (di->dupstr == di->kv.str)
        di->dupstr = solv_memdup(di->dupstr, di->dupstrn);
      else
	{
	  di->dupstr = 0;
	  di->dupstrn = 0;
	}
    }
  memset(&di->matcher, 0, sizeof(di->matcher));
  if (from->matcher.match)
    datamatcher_init(&di->matcher, from->matcher.match, from->matcher.flags);
  if (di->nparents)
    {
      /* fix pointers */
      int i;
      for (i = 1; i < di->nparents; i++)
	di->parents[i].kv.parent = &di->parents[i - 1].kv;
      di->kv.parent = &di->parents[di->nparents - 1].kv;
    }
  if (di->oldkeyskip)
    di->oldkeyskip = solv_memdup2(di->oldkeyskip, 3 + di->oldkeyskip[0], sizeof(Id));
  if (di->keyskip)
    di->keyskip = di->oldkeyskip;
}

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_set_deltalocation(Repodata *data, Id handle, int medianr, const char *dir, const char *file)
{
  int l = 0;
  const char *evr, *suf, *s;

  if (!dir)
    {
      if ((dir = strrchr(file, '/')) != 0)
	{
          l = dir - file;
	  dir = file;
	  file = dir + l + 1;
	  if (!l)
	    l++;
	}
    }
  else
    l = strlen(dir);
  if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
    {
      dir += 2;
      l -= 2;
    }
  if (l == 1 && dir[0] == '.')
    l = 0;
  if (dir && l)
    repodata_set_poolstrn(data, handle, DELTA_LOCATION_DIR, dir, l);
  evr = strchr(file, '-');
  if (evr)
    {
      for (s = evr - 1; s > file; s--)
	if (*s == '-')
	  {
	    evr = s;
	    break;
	  }
    }
  suf = strrchr(file, '.');
  if (suf)
    {
      for (s = suf - 1; s > file; s--)
	if (*s == '.')
	  {
	    suf = s;
	    break;
	  }
      if (!strcmp(suf, ".delta.rpm") || !strcmp(suf, ".patch.rpm"))
	{
	  /* We accept one more item as suffix.  */
	  for (s = suf - 1; s > file; s--)
	    if (*s == '.')
	      {
		suf = s;
	        break;
	      }
	}
    }
  if (!evr)
    suf = 0;
  if (suf && evr && suf < evr)
    suf = 0;
  repodata_set_poolstrn(data, handle, DELTA_LOCATION_NAME, file, evr ? evr - file : strlen(file));
  if (evr)
    repodata_set_poolstrn(data, handle, DELTA_LOCATION_EVR, evr + 1, suf ? suf - evr - 1: strlen(evr + 1));
  if (suf)
    repodata_set_poolstr(data, handle, DELTA_LOCATION_SUFFIX, suf + 1);
}


repodata_set_poolstrn(Repodata *data, Id solvid, Id keyname, const char *str, int l)
{
  Id id;
  if (data->localpool)
    id = stringpool_strn2id(&data->spool, str, l, 1);
  else
    id = pool_strn2id(data->repo->pool, str, l, 1);
  repodata_set_id(data, solvid, keyname, id);
}


repodata_set_num(Repodata *data, Id solvid, Id keyname, unsigned long long num)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_NUM;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  if (num >= 0x80000000)
    {
      data->attrnum64data = solv_extend(data->attrnum64data, data->attrnum64datalen, 1, sizeof(unsigned long long), REPODATA_ATTRNUM64DATA_BLOCK);
      data->attrnum64data[data->attrnum64datalen] = num;
      num = 0x80000000 | data->attrnum64datalen++;
    }
  repodata_set(data, solvid, &key, (Id)num);
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct subschema_data {
  void *cbdata;
  Id solvid;
  KeyValue *parent;
}

#define REPODATA_ATTRNUM64DATA_BLOCK 15

struct extdata {
  unsigned char *buf;
  int len;
}


dataiterator_setpos(Dataiterator *di)
{
  if (di->kv.eof == 2)
    {
      pool_clear_pos(di->pool);
      return;
    }
  di->pool->pos.solvid = di->solvid;
  di->pool->pos.repo = di->repo;
  di->pool->pos.repodataid = di->data - di->repo->repodata;
  di->pool->pos.schema = di->kv.id;
  di->pool->pos.dp = (unsigned char *)di->kv.str - di->data->incoredata;
}


maybe_load_repodata_stub(Repodata *data, Id keyname)
{
  if (data->state != REPODATA_STUB)
    {
      data->state = REPODATA_ERROR;
      return 0;
    }
  if (keyname)
    {
      int i;
      for (i = 1; i < data->nkeys; i++)
	if (keyname == data->keys[i].name)
	  break;
      if (i == data->nkeys)
	return 0;
    }
  repodata_load(data);
  return data->state == REPODATA_AVAILABLE ? 1 : 0;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_search_arrayelement(Repodata *data, Id solvid, Id keyname, int flags, KeyValue *kv, int (*callback)(void *cbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv), void *cbdata)
{
  repodata_search_keyskip(data, solvid, keyname, flags | SEARCH_SUBSCHEMA, (Id *)kv, callback, cbdata);
}


repodata_set_void(Repodata *data, Id solvid, Id keyname)
{
  Repokey key;
  key.name = keyname;
  key.type = REPOKEY_TYPE_VOID;
  key.size = 0;
  key.storage = KEY_STORAGE_INCORE;
  repodata_set(data, solvid, &key, 0);
}


datamatcher_match(Datamatcher *ma, const char *str)
{
  int l;
  switch ((ma->flags & SEARCH_STRINGMASK))
    {
    case SEARCH_SUBSTRING:
      if (ma->flags & SEARCH_NOCASE)
	return strcasestr(str, ma->match) != 0;
      else
	return strstr(str, ma->match) != 0;
    case SEARCH_STRING:
      if (ma->flags & SEARCH_NOCASE)
	return !strcasecmp(ma->match, str);
      else
	return !strcmp(ma->match, str);
    case SEARCH_STRINGSTART:
      if (ma->flags & SEARCH_NOCASE)
        return !strncasecmp(ma->match, str, strlen(ma->match));
      else
        return !strncmp(ma->match, str, strlen(ma->match));
    case SEARCH_STRINGEND:
      l = strlen(str) - strlen(ma->match);
      if (l < 0)
	return 0;
      if (ma->flags & SEARCH_NOCASE)
	return !strcasecmp(ma->match, str + l);
      else
	return !strcmp(ma->match, str + l);
    case SEARCH_GLOB:
      return !fnmatch(ma->match, str, (ma->flags & SEARCH_NOCASE) ? FNM_CASEFOLD : 0);
    case SEARCH_REGEX:
      return !regexec((const regex_t *)ma->matchdata, str, 0, NULL, 0);
    default:
      return 0;
    }
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);

struct dircache {
  Id ids[DIRCACHE_SIZE];
  char str[(DIRCACHE_SIZE * (DIRCACHE_SIZE - 1)) / 2];
}

struct extdata {
  unsigned char *buf;
  int len;
}


repodata_load(Repodata *data)
{
  if (data->state != REPODATA_STUB)
    return;
  if (data->loadcallback)
    data->loadcallback(data);
  else
    data->state = REPODATA_ERROR;
}

static unsigned char *data_skip_key(Repodata *data, unsigned char *dp, Repokey *key);


repodata_set_location(Repodata *data, Id solvid, int medianr, const char *dir, const char *file)
{
  Pool *pool = data->repo->pool;
  Solvable *s;
  const char *str, *fp;
  int l = 0;

  if (medianr)
    repodata_set_constant(data, solvid, SOLVABLE_MEDIANR, medianr);
  if (!dir)
    {
      if ((dir = strrchr(file, '/')) != 0)
	{
          l = dir - file;
	  dir = file;
	  file = dir + l + 1;
	  if (!l)
	    l++;
	}
    }
  else
    l = strlen(dir);
  if (l >= 2 && dir[0] == '.' && dir[1] == '/' && (l == 2 || dir[2] != '/'))
    {
      dir += 2;
      l -= 2;
    }
  if (l == 1 && dir[0] == '.')
    l = 0;
  s = pool->solvables + solvid;
  if (dir && l)
    {
      str = pool_id2str(pool, s->arch);
      if (!strncmp(dir, str, l) && !str[l])
	repodata_set_void(data, solvid, SOLVABLE_MEDIADIR);
      else
	repodata_set_strn(data, solvid, SOLVABLE_MEDIADIR, dir, l);
    }
  fp = file;
  str = pool_id2str(pool, s->name);
  l = strlen(str);
  if ((!l || !strncmp(fp, str, l)) && fp[l] == '-')
    {
      fp += l + 1;
      str = evrid2vrstr(pool, s->evr);
      l = strlen(str);
      if ((!l || !strncmp(fp, str, l)) && fp[l] == '.')
	{
	  fp += l + 1;
	  str = pool_id2str(pool, s->arch);
	  l = strlen(str);
	  if ((!l || !strncmp(fp, str, l)) && !strcmp(fp + l, ".rpm"))
	    {
	      repodata_set_void(data, solvid, SOLVABLE_MEDIAFILE);
	      return;
	    }
	}
    }
  repodata_set_str(data, solvid, SOLVABLE_MEDIAFILE, file);
}


data_skip_key(Repodata *data, unsigned char *dp, Repokey *key)
{
  int nentries, schema;
  switch(key->type)
    {
    case REPOKEY_TYPE_FIXARRAY:
      dp = data_read_id(dp, &nentries);
      if (!nentries)
	return dp;
      dp = data_read_id(dp, &schema);
      while (nentries--)
	dp = data_skip_schema(data, dp, schema);
      return dp;
    case REPOKEY_TYPE_FLEXARRAY:
      dp = data_read_id(dp, &nentries);
      while (nentries--)
	{
	  dp = data_read_id(dp, &schema);
	  dp = data_skip_schema(data, dp, schema);
	}
      return dp;
    default:
      if (key->storage == KEY_STORAGE_INCORE)
        dp = data_skip(dp, key->type);
      else if (key->storage == KEY_STORAGE_VERTICAL_OFFSET)
	{
	  dp = data_skip(dp, REPOKEY_TYPE_ID);
	  dp = data_skip(dp, REPOKEY_TYPE_ID);
	}
      return dp;
    }
}