mkspell(
    int		fcount,
    char_u	**fnames,
    int		ascii,		    /* -ascii argument given */
    int		over_write,	    /* overwrite existing output file */
    int		added_word)	    /* invoked through "zg" */
{
    char_u	*fname = NULL;
    char_u	*wfname;
    char_u	**innames;
    int		incount;
    afffile_T	*(afile[8]);
    int		i;
    int		len;
    stat_T	st;
    int		error = FALSE;
    spellinfo_T spin;

    vim_memset(&spin, 0, sizeof(spin));
    spin.si_verbose = !added_word;
    spin.si_ascii = ascii;
    spin.si_followup = TRUE;
    spin.si_rem_accents = TRUE;
    ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20);
    ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20);
    ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20);
    ga_init2(&spin.si_map, (int)sizeof(char_u), 100);
    ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20);
    ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50);
    hash_init(&spin.si_commonwords);
    spin.si_newcompID = 127;	/* start compound ID at first maximum */

    /* default: fnames[0] is output file, following are input files */
    innames = &fnames[1];
    incount = fcount - 1;

    wfname = alloc(MAXPATHL);
    if (wfname == NULL)
	return;

    if (fcount >= 1)
    {
	len = (int)STRLEN(fnames[0]);
	if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0)
	{
	    /* For ":mkspell path/en.latin1.add" output file is
	     * "path/en.latin1.add.spl". */
	    innames = &fnames[0];
	    incount = 1;
	    vim_snprintf((char *)wfname, MAXPATHL, "%s.spl", fnames[0]);
	}
	else if (fcount == 1)
	{
	    /* For ":mkspell path/vim" output file is "path/vim.latin1.spl". */
	    innames = &fnames[0];
	    incount = 1;
	    vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL,
		  fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc());
	}
	else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0)
	{
	    /* Name ends in ".spl", use as the file name. */
	    vim_strncpy(wfname, fnames[0], MAXPATHL - 1);
	}
	else
	    /* Name should be language, make the file name from it. */
	    vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL,
		  fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc());

	/* Check for .ascii.spl. */
	if (strstr((char *)gettail(wfname), SPL_FNAME_ASCII) != NULL)
	    spin.si_ascii = TRUE;

	/* Check for .add.spl. */
	if (strstr((char *)gettail(wfname), SPL_FNAME_ADD) != NULL)
	    spin.si_add = TRUE;
    }

    if (incount <= 0)
	EMSG(_(e_invarg));	/* need at least output and input names */
    else if (vim_strchr(gettail(wfname), '_') != NULL)
	EMSG(_("E751: Output file name must not have region name"));
    else if (incount > 8)
	EMSG(_("E754: Only up to 8 regions supported"));
    else
    {
	/* Check for overwriting before doing things that may take a lot of
	 * time. */
	if (!over_write && mch_stat((char *)wfname, &st) >= 0)
	{
	    EMSG(_(e_exists));
	    goto theend;
	}
	if (mch_isdir(wfname))
	{
	    EMSG2(_(e_isadir2), wfname);
	    goto theend;
	}

	fname = alloc(MAXPATHL);
	if (fname == NULL)
	    goto theend;

	/*
	 * Init the aff and dic pointers.
	 * Get the region names if there are more than 2 arguments.
	 */
	for (i = 0; i < incount; ++i)
	{
	    afile[i] = NULL;

	    if (incount > 1)
	    {
		len = (int)STRLEN(innames[i]);
		if (STRLEN(gettail(innames[i])) < 5
						|| innames[i][len - 3] != '_')
		{
		    EMSG2(_("E755: Invalid region in %s"), innames[i]);
		    goto theend;
		}
		spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]);
		spin.si_region_name[i * 2 + 1] =
					     TOLOWER_ASC(innames[i][len - 1]);
	    }
	}
	spin.si_region_count = incount;

	spin.si_foldroot = wordtree_alloc(&spin);
	spin.si_keeproot = wordtree_alloc(&spin);
	spin.si_prefroot = wordtree_alloc(&spin);
	if (spin.si_foldroot == NULL
		|| spin.si_keeproot == NULL
		|| spin.si_prefroot == NULL)
	{
	    free_blocks(spin.si_blocks);
	    goto theend;
	}

	/* When not producing a .add.spl file clear the character table when
	 * we encounter one in the .aff file.  This means we dump the current
	 * one in the .spl file if the .aff file doesn't define one.  That's
	 * better than guessing the contents, the table will match a
	 * previously loaded spell file. */
	if (!spin.si_add)
	    spin.si_clear_chartab = TRUE;

	/*
	 * Read all the .aff and .dic files.
	 * Text is converted to 'encoding'.
	 * Words are stored in the case-folded and keep-case trees.
	 */
	for (i = 0; i < incount && !error; ++i)
	{
	    spin.si_conv.vc_type = CONV_NONE;
	    spin.si_region = 1 << i;

	    vim_snprintf((char *)fname, MAXPATHL, "%s.aff", innames[i]);
	    if (mch_stat((char *)fname, &st) >= 0)
	    {
		/* Read the .aff file.  Will init "spin->si_conv" based on the
		 * "SET" line. */
		afile[i] = spell_read_aff(&spin, fname);
		if (afile[i] == NULL)
		    error = TRUE;
		else
		{
		    /* Read the .dic file and store the words in the trees. */
		    vim_snprintf((char *)fname, MAXPATHL, "%s.dic",
								  innames[i]);
		    if (spell_read_dic(&spin, fname, afile[i]) == FAIL)
			error = TRUE;
		}
	    }
	    else
	    {
		/* No .aff file, try reading the file as a word list.  Store
		 * the words in the trees. */
		if (spell_read_wordfile(&spin, innames[i]) == FAIL)
		    error = TRUE;
	    }

#ifdef FEAT_MBYTE
	    /* Free any conversion stuff. */
	    convert_setup(&spin.si_conv, NULL, NULL);
#endif
	}

	if (spin.si_compflags != NULL && spin.si_nobreak)
	    MSG(_("Warning: both compounding and NOBREAK specified"));

	if (!error && !got_int)
	{
	    /*
	     * Combine tails in the tree.
	     */
	    spell_message(&spin, (char_u *)_(msg_compressing));
	    wordtree_compress(&spin, spin.si_foldroot);
	    wordtree_compress(&spin, spin.si_keeproot);
	    wordtree_compress(&spin, spin.si_prefroot);
	}

	if (!error && !got_int)
	{
	    /*
	     * Write the info in the spell file.
	     */
	    vim_snprintf((char *)IObuff, IOSIZE,
				      _("Writing spell file %s ..."), wfname);
	    spell_message(&spin, IObuff);

	    error = write_vim_spell(&spin, wfname) == FAIL;

	    spell_message(&spin, (char_u *)_("Done!"));
	    vim_snprintf((char *)IObuff, IOSIZE,
		 _("Estimated runtime memory use: %d bytes"), spin.si_memtot);
	    spell_message(&spin, IObuff);

	    /*
	     * If the file is loaded need to reload it.
	     */
	    if (!error)
		spell_reload_one(wfname, added_word);
	}

	/* Free the allocated memory. */
	ga_clear(&spin.si_rep);
	ga_clear(&spin.si_repsal);
	ga_clear(&spin.si_sal);
	ga_clear(&spin.si_map);
	ga_clear(&spin.si_comppat);
	ga_clear(&spin.si_prefcond);
	hash_clear_all(&spin.si_commonwords, 0);

	/* Free the .aff file structures. */
	for (i = 0; i < incount; ++i)
	    if (afile[i] != NULL)
		spell_free_aff(afile[i]);

	/* Free all the bits and pieces at once. */
	free_blocks(spin.si_blocks);

	/*
	 * If there is soundfolding info and no NOSUGFILE item create the
	 * .sug file with the soundfolded word trie.
	 */
	if (spin.si_sugtime != 0 && !error && !got_int)
	    spell_make_sugfile(&spin, wfname);

    }

theend:
    vim_free(fname);
    vim_free(wfname);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SAL_F0LLOWUP		1

#define SAL_COLLAPSE		2

#define SAL_REM_ACCENTS		4

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define VIMSPELLMAGICL 8

#define VIMSPELLVERSION 50

#define SN_REGION	0	/* <regionname> section */

#define SN_CHARFLAGS	1	/* charflags section */

#define SN_MIDWORD	2	/* <midword> section */

#define SN_PREFCOND	3	/* <prefcond> section */

#define SN_REP		4	/* REP items section */

#define SN_SAL		5	/* SAL items section */

#define SN_SOFO		6	/* soundfolding section */

#define SN_MAP		7	/* MAP items section */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SYLLABLE	9	/* syllable section */

#define SN_NOBREAK	10	/* NOBREAK section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_REPSAL	12	/* REPSAL items section */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SN_END		255	/* end of sections */

#define SNF_REQUIRED	1	/* <sectionflags>: required section */

#define CF_WORD		0x01

#define CF_UPPER	0x02

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);


set_sal_first(slang_T *lp)
{
    salfirst_T	*sfirst;
    int		i;
    salitem_T	*smp;
    int		c;
    garray_T	*gap = &lp->sl_sal;

    sfirst = lp->sl_sal_first;
    for (i = 0; i < 256; ++i)
	sfirst[i] = -1;
    smp = (salitem_T *)gap->ga_data;
    for (i = 0; i < gap->ga_len; ++i)
    {
#ifdef FEAT_MBYTE
	if (has_mbyte)
	    /* Use the lowest byte of the first character.  For latin1 it's
	     * the character, for other encodings it should differ for most
	     * characters. */
	    c = *smp[i].sm_lead_w & 0xff;
	else
#endif
	    c = *smp[i].sm_lead;
	if (sfirst[c] == -1)
	{
	    sfirst[c] = i;
#ifdef FEAT_MBYTE
	    if (has_mbyte)
	    {
		int		n;

		/* Make sure all entries with this byte are following each
		 * other.  Move the ones that are in the wrong position.  Do
		 * keep the same ordering! */
		while (i + 1 < gap->ga_len
				       && (*smp[i + 1].sm_lead_w & 0xff) == c)
		    /* Skip over entry with same index byte. */
		    ++i;

		for (n = 1; i + n < gap->ga_len; ++n)
		    if ((*smp[i + n].sm_lead_w & 0xff) == c)
		    {
			salitem_T  tsal;

			/* Move entry with same index byte after the entries
			 * we already found. */
			++i;
			--n;
			tsal = smp[i + n];
			mch_memmove(smp + i + 1, smp + i,
						       sizeof(salitem_T) * n);
			smp[i] = tsal;
		    }
	    }
#endif
	}
    }
}

patch 8.0.0322: possible overflow with corrupted spell file

Problem:    Possible overflow with spell file where the tree length is
            corrupted.
Solution:   Check for an invalid length (suggested by shqking)

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);


spell_add_word(
    char_u	*word,
    int		len,
    int		bad,
    int		idx,	    /* "zG" and "zW": zero, otherwise index in
			       'spellfile' */
    int		undo)	    /* TRUE for "zug", "zuG", "zuw" and "zuW" */
{
    FILE	*fd = NULL;
    buf_T	*buf = NULL;
    int		new_spf = FALSE;
    char_u	*fname;
    char_u	*fnamebuf = NULL;
    char_u	line[MAXWLEN * 2];
    long	fpos, fpos_next = 0;
    int		i;
    char_u	*spf;

    if (idx == 0)	    /* use internal wordlist */
    {
	if (int_wordlist == NULL)
	{
	    int_wordlist = vim_tempname('s', FALSE);
	    if (int_wordlist == NULL)
		return;
	}
	fname = int_wordlist;
    }
    else
    {
	/* If 'spellfile' isn't set figure out a good default value. */
	if (*curwin->w_s->b_p_spf == NUL)
	{
	    init_spellfile();
	    new_spf = TRUE;
	}

	if (*curwin->w_s->b_p_spf == NUL)
	{
	    EMSG2(_(e_notset), "spellfile");
	    return;
	}
	fnamebuf = alloc(MAXPATHL);
	if (fnamebuf == NULL)
	    return;

	for (spf = curwin->w_s->b_p_spf, i = 1; *spf != NUL; ++i)
	{
	    copy_option_part(&spf, fnamebuf, MAXPATHL, ",");
	    if (i == idx)
		break;
	    if (*spf == NUL)
	    {
		EMSGN(_("E765: 'spellfile' does not have %ld entries"), idx);
		vim_free(fnamebuf);
		return;
	    }
	}

	/* Check that the user isn't editing the .add file somewhere. */
	buf = buflist_findname_exp(fnamebuf);
	if (buf != NULL && buf->b_ml.ml_mfp == NULL)
	    buf = NULL;
	if (buf != NULL && bufIsChanged(buf))
	{
	    EMSG(_(e_bufloaded));
	    vim_free(fnamebuf);
	    return;
	}

	fname = fnamebuf;
    }

    if (bad || undo)
    {
	/* When the word appears as good word we need to remove that one,
	 * since its flags sort before the one with WF_BANNED. */
	fd = mch_fopen((char *)fname, "r");
	if (fd != NULL)
	{
	    while (!vim_fgets(line, MAXWLEN * 2, fd))
	    {
		fpos = fpos_next;
		fpos_next = ftell(fd);
		if (STRNCMP(word, line, len) == 0
			&& (line[len] == '/' || line[len] < ' '))
		{
		    /* Found duplicate word.  Remove it by writing a '#' at
		     * the start of the line.  Mixing reading and writing
		     * doesn't work for all systems, close the file first. */
		    fclose(fd);
		    fd = mch_fopen((char *)fname, "r+");
		    if (fd == NULL)
			break;
		    if (fseek(fd, fpos, SEEK_SET) == 0)
		    {
			fputc('#', fd);
			if (undo)
			{
			    home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
			    smsg((char_u *)_("Word '%.*s' removed from %s"),
							 len, word, NameBuff);
			}
		    }
		    fseek(fd, fpos_next, SEEK_SET);
		}
	    }
	    if (fd != NULL)
		fclose(fd);
	}
    }

    if (!undo)
    {
	fd = mch_fopen((char *)fname, "a");
	if (fd == NULL && new_spf)
	{
	    char_u *p;

	    /* We just initialized the 'spellfile' option and can't open the
	     * file.  We may need to create the "spell" directory first.  We
	     * already checked the runtime directory is writable in
	     * init_spellfile(). */
	    if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname)
	    {
		int c = *p;

		/* The directory doesn't exist.  Try creating it and opening
		 * the file again. */
		*p = NUL;
		vim_mkdir(fname, 0755);
		*p = c;
		fd = mch_fopen((char *)fname, "a");
	    }
	}

	if (fd == NULL)
	    EMSG2(_(e_notopen), fname);
	else
	{
	    if (bad)
		fprintf(fd, "%.*s/!\n", len, word);
	    else
		fprintf(fd, "%.*s\n", len, word);
	    fclose(fd);

	    home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE);
	    smsg((char_u *)_("Word '%.*s' added to %s"), len, word, NameBuff);
	}
    }

    if (fd != NULL)
    {
	/* Update the .add.spl file. */
	mkspell(1, &fname, FALSE, TRUE, TRUE);

	/* If the .add file is edited somewhere, reload it. */
	if (buf != NULL)
	    buf_reload(buf, buf->b_orig_mode);

	redraw_all_later(SOME_VALID);
    }
    vim_free(fnamebuf);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


aff_check_string(char_u *spinval, char_u *affval, char *name)
{
    if (spinval != NULL && STRCMP(spinval, affval) != 0)
	smsg((char_u *)_("%s value differs from what is used in another .aff file"), name);
}

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


spell_make_sugfile(spellinfo_T *spin, char_u *wfname)
{
    char_u	*fname = NULL;
    int		len;
    slang_T	*slang;
    int		free_slang = FALSE;

    /*
     * Read back the .spl file that was written.  This fills the required
     * info for soundfolding.  This also uses less memory than the
     * pointer-linked version of the trie.  And it avoids having two versions
     * of the code for the soundfolding stuff.
     * It might have been done already by spell_reload_one().
     */
    for (slang = first_lang; slang != NULL; slang = slang->sl_next)
	if (fullpathcmp(wfname, slang->sl_fname, FALSE) == FPC_SAME)
	    break;
    if (slang == NULL)
    {
	spell_message(spin, (char_u *)_("Reading back spell file..."));
	slang = spell_load_file(wfname, NULL, NULL, FALSE);
	if (slang == NULL)
	    return;
	free_slang = TRUE;
    }

    /*
     * Clear the info in "spin" that is used.
     */
    spin->si_blocks = NULL;
    spin->si_blocks_cnt = 0;
    spin->si_compress_cnt = 0;	    /* will stay at 0 all the time*/
    spin->si_free_count = 0;
    spin->si_first_free = NULL;
    spin->si_foldwcount = 0;

    /*
     * Go through the trie of good words, soundfold each word and add it to
     * the soundfold trie.
     */
    spell_message(spin, (char_u *)_("Performing soundfolding..."));
    if (sug_filltree(spin, slang) == FAIL)
	goto theend;

    /*
     * Create the table which links each soundfold word with a list of the
     * good words it may come from.  Creates buffer "spin->si_spellbuf".
     * This also removes the wordnr from the NUL byte entries to make
     * compression possible.
     */
    if (sug_maketable(spin) == FAIL)
	goto theend;

    smsg((char_u *)_("Number of words after soundfolding: %ld"),
				 (long)spin->si_spellbuf->b_ml.ml_line_count);

    /*
     * Compress the soundfold trie.
     */
    spell_message(spin, (char_u *)_(msg_compressing));
    wordtree_compress(spin, spin->si_foldroot);

    /*
     * Write the .sug file.
     * Make the file name by changing ".spl" to ".sug".
     */
    fname = alloc(MAXPATHL);
    if (fname == NULL)
	goto theend;
    vim_strncpy(fname, wfname, MAXPATHL - 1);
    len = (int)STRLEN(fname);
    fname[len - 2] = 'u';
    fname[len - 1] = 'g';
    sug_write(spin, fname);

theend:
    vim_free(fname);
    if (free_slang)
	slang_free(slang);
    free_blocks(spin->si_blocks);
    close_spellbuf(spin->si_spellbuf);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SOFO		6	/* soundfolding section */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SNF_REQUIRED	1	/* <sectionflags>: required section */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


suggest_load_files(void)
{
    langp_T	*lp;
    int		lpi;
    slang_T	*slang;
    char_u	*dotp;
    FILE	*fd;
    char_u	buf[MAXWLEN];
    int		i;
    time_t	timestamp;
    int		wcount;
    int		wordnr;
    garray_T	ga;
    int		c;

    /* Do this for all languages that support sound folding. */
    for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi)
    {
	lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi);
	slang = lp->lp_slang;
	if (slang->sl_sugtime != 0 && !slang->sl_sugloaded)
	{
	    /* Change ".spl" to ".sug" and open the file.  When the file isn't
	     * found silently skip it.  Do set "sl_sugloaded" so that we
	     * don't try again and again. */
	    slang->sl_sugloaded = TRUE;

	    dotp = vim_strrchr(slang->sl_fname, '.');
	    if (dotp == NULL || fnamecmp(dotp, ".spl") != 0)
		continue;
	    STRCPY(dotp, ".sug");
	    fd = mch_fopen((char *)slang->sl_fname, "r");
	    if (fd == NULL)
		goto nextone;

	    /*
	     * <SUGHEADER>: <fileID> <versionnr> <timestamp>
	     */
	    for (i = 0; i < VIMSUGMAGICL; ++i)
		buf[i] = getc(fd);			/* <fileID> */
	    if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0)
	    {
		EMSG2(_("E778: This does not look like a .sug file: %s"),
							     slang->sl_fname);
		goto nextone;
	    }
	    c = getc(fd);				/* <versionnr> */
	    if (c < VIMSUGVERSION)
	    {
		EMSG2(_("E779: Old .sug file, needs to be updated: %s"),
							     slang->sl_fname);
		goto nextone;
	    }
	    else if (c > VIMSUGVERSION)
	    {
		EMSG2(_("E780: .sug file is for newer version of Vim: %s"),
							     slang->sl_fname);
		goto nextone;
	    }

	    /* Check the timestamp, it must be exactly the same as the one in
	     * the .spl file.  Otherwise the word numbers won't match. */
	    timestamp = get8ctime(fd);			/* <timestamp> */
	    if (timestamp != slang->sl_sugtime)
	    {
		EMSG2(_("E781: .sug file doesn't match .spl file: %s"),
							     slang->sl_fname);
		goto nextone;
	    }

	    /*
	     * <SUGWORDTREE>: <wordtree>
	     * Read the trie with the soundfolded words.
	     */
	    if (spell_read_tree(fd, &slang->sl_sbyts, &slang->sl_sidxs,
							       FALSE, 0) != 0)
	    {
someerror:
		EMSG2(_("E782: error while reading .sug file: %s"),
							     slang->sl_fname);
		slang_clear_sug(slang);
		goto nextone;
	    }

	    /*
	     * <SUGTABLE>: <sugwcount> <sugline> ...
	     *
	     * Read the table with word numbers.  We use a file buffer for
	     * this, because it's so much like a file with lines.  Makes it
	     * possible to swap the info and save on memory use.
	     */
	    slang->sl_sugbuf = open_spellbuf();
	    if (slang->sl_sugbuf == NULL)
		goto someerror;
							    /* <sugwcount> */
	    wcount = get4c(fd);
	    if (wcount < 0)
		goto someerror;

	    /* Read all the wordnr lists into the buffer, one NUL terminated
	     * list per line. */
	    ga_init2(&ga, 1, 100);
	    for (wordnr = 0; wordnr < wcount; ++wordnr)
	    {
		ga.ga_len = 0;
		for (;;)
		{
		    c = getc(fd);			    /* <sugline> */
		    if (c < 0 || ga_grow(&ga, 1) == FAIL)
			goto someerror;
		    ((char_u *)ga.ga_data)[ga.ga_len++] = c;
		    if (c == NUL)
			break;
		}
		if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr,
					 ga.ga_data, ga.ga_len, TRUE) == FAIL)
		    goto someerror;
	    }
	    ga_clear(&ga);

	    /*
	     * Need to put word counts in the word tries, so that we can find
	     * a word by its number.
	     */
	    tree_count_words(slang->sl_fbyts, slang->sl_fidxs);
	    tree_count_words(slang->sl_sbyts, slang->sl_sidxs);

nextone:
	    if (fd != NULL)
		fclose(fd);
	    STRCPY(dotp, ".spl");
	}
    }
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


read_cnt_string(FILE *fd, int cnt_bytes, int *cntp)
{
    int		cnt = 0;
    int		i;
    char_u	*str;

    /* read the length bytes, MSB first */
    for (i = 0; i < cnt_bytes; ++i)
	cnt = (cnt << 8) + getc(fd);
    if (cnt < 0)
    {
	*cntp = SP_TRUNCERROR;
	return NULL;
    }
    *cntp = cnt;
    if (cnt == 0)
	return NULL;	    /* nothing to read, return NULL */

    str = read_string(fd, cnt);
    if (str == NULL)
	*cntp = SP_OTHERERROR;
    return str;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


read_prefcond_section(FILE *fd, slang_T *lp)
{
    int		cnt;
    int		i;
    int		n;
    char_u	*p;
    char_u	buf[MAXWLEN + 1];

    /* <prefcondcnt> <prefcond> ... */
    cnt = get2c(fd);					/* <prefcondcnt> */
    if (cnt <= 0)
	return SP_FORMERROR;

    lp->sl_prefprog = (regprog_T **)alloc_clear(
					 (unsigned)sizeof(regprog_T *) * cnt);
    if (lp->sl_prefprog == NULL)
	return SP_OTHERERROR;
    lp->sl_prefixcnt = cnt;

    for (i = 0; i < cnt; ++i)
    {
	/* <prefcond> : <condlen> <condstr> */
	n = getc(fd);					/* <condlen> */
	if (n < 0 || n >= MAXWLEN)
	    return SP_FORMERROR;

	/* When <condlen> is zero we have an empty condition.  Otherwise
	 * compile the regexp program used to check for the condition. */
	if (n > 0)
	{
	    buf[0] = '^';	    /* always match at one position only */
	    p = buf + 1;
	    while (n-- > 0)
		*p++ = getc(fd);			/* <condstr> */
	    *p = NUL;
	    lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING);
	}
    }
    return 0;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_PREFCOND	3	/* <prefcond> section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);


read_compound(FILE *fd, slang_T *slang, int len)
{
    int		todo = len;
    int		c;
    int		atstart;
    char_u	*pat;
    char_u	*pp;
    char_u	*cp;
    char_u	*ap;
    char_u	*crp;
    int		cnt;
    garray_T	*gap;

    if (todo < 2)
	return SP_FORMERROR;	/* need at least two bytes */

    --todo;
    c = getc(fd);					/* <compmax> */
    if (c < 2)
	c = MAXWLEN;
    slang->sl_compmax = c;

    --todo;
    c = getc(fd);					/* <compminlen> */
    if (c < 1)
	c = 0;
    slang->sl_compminlen = c;

    --todo;
    c = getc(fd);					/* <compsylmax> */
    if (c < 1)
	c = MAXWLEN;
    slang->sl_compsylmax = c;

    c = getc(fd);					/* <compoptions> */
    if (c != 0)
	ungetc(c, fd);	    /* be backwards compatible with Vim 7.0b */
    else
    {
	--todo;
	c = getc(fd);	    /* only use the lower byte for now */
	--todo;
	slang->sl_compoptions = c;

	gap = &slang->sl_comppat;
	c = get2c(fd);					/* <comppatcount> */
	todo -= 2;
	ga_init2(gap, sizeof(char_u *), c);
	if (ga_grow(gap, c) == OK)
	    while (--c >= 0)
	    {
		((char_u **)(gap->ga_data))[gap->ga_len++] =
						 read_cnt_string(fd, 1, &cnt);
					    /* <comppatlen> <comppattext> */
		if (cnt < 0)
		    return cnt;
		todo -= cnt + 1;
	    }
    }
    if (todo < 0)
	return SP_FORMERROR;

    /* Turn the COMPOUNDRULE items into a regexp pattern:
     * "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$".
     * Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes.
     * Conversion to utf-8 may double the size. */
    c = todo * 2 + 7;
#ifdef FEAT_MBYTE
    if (enc_utf8)
	c += todo * 2;
#endif
    pat = alloc((unsigned)c);
    if (pat == NULL)
	return SP_OTHERERROR;

    /* We also need a list of all flags that can appear at the start and one
     * for all flags. */
    cp = alloc(todo + 1);
    if (cp == NULL)
    {
	vim_free(pat);
	return SP_OTHERERROR;
    }
    slang->sl_compstartflags = cp;
    *cp = NUL;

    ap = alloc(todo + 1);
    if (ap == NULL)
    {
	vim_free(pat);
	return SP_OTHERERROR;
    }
    slang->sl_compallflags = ap;
    *ap = NUL;

    /* And a list of all patterns in their original form, for checking whether
     * compounding may work in match_compoundrule().  This is freed when we
     * encounter a wildcard, the check doesn't work then. */
    crp = alloc(todo + 1);
    slang->sl_comprules = crp;

    pp = pat;
    *pp++ = '^';
    *pp++ = '\\';
    *pp++ = '(';

    atstart = 1;
    while (todo-- > 0)
    {
	c = getc(fd);					/* <compflags> */
	if (c == EOF)
	{
	    vim_free(pat);
	    return SP_TRUNCERROR;
	}

	/* Add all flags to "sl_compallflags". */
	if (vim_strchr((char_u *)"?*+[]/", c) == NULL
		&& !byte_in_str(slang->sl_compallflags, c))
	{
	    *ap++ = c;
	    *ap = NUL;
	}

	if (atstart != 0)
	{
	    /* At start of item: copy flags to "sl_compstartflags".  For a
	     * [abc] item set "atstart" to 2 and copy up to the ']'. */
	    if (c == '[')
		atstart = 2;
	    else if (c == ']')
		atstart = 0;
	    else
	    {
		if (!byte_in_str(slang->sl_compstartflags, c))
		{
		    *cp++ = c;
		    *cp = NUL;
		}
		if (atstart == 1)
		    atstart = 0;
	    }
	}

	/* Copy flag to "sl_comprules", unless we run into a wildcard. */
	if (crp != NULL)
	{
	    if (c == '?' || c == '+' || c == '*')
	    {
		vim_free(slang->sl_comprules);
		slang->sl_comprules = NULL;
		crp = NULL;
	    }
	    else
		*crp++ = c;
	}

	if (c == '/')	    /* slash separates two items */
	{
	    *pp++ = '\\';
	    *pp++ = '|';
	    atstart = 1;
	}
	else		    /* normal char, "[abc]" and '*' are copied as-is */
	{
	    if (c == '?' || c == '+' || c == '~')
		*pp++ = '\\';	    /* "a?" becomes "a\?", "a+" becomes "a\+" */
#ifdef FEAT_MBYTE
	    if (enc_utf8)
		pp += mb_char2bytes(c, pp);
	    else
#endif
		*pp++ = c;
	}
    }

    *pp++ = '\\';
    *pp++ = ')';
    *pp++ = '$';
    *pp = NUL;

    if (crp != NULL)
	*crp = NUL;

    slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT);
    vim_free(pat);
    if (slang->sl_compprog == NULL)
	return SP_FORMERROR;

    return 0;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_REP		4	/* REP items section */

#define SN_SAL		5	/* SAL items section */

#define SN_MAP		7	/* MAP items section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_REPSAL	12	/* REPSAL items section */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


get_wordnode(spellinfo_T *spin)
{
    wordnode_T *n;

    if (spin->si_first_free == NULL)
	n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE);
    else
    {
	n = spin->si_first_free;
	spin->si_first_free = n->wn_child;
	vim_memset(n, 0, sizeof(wordnode_T));
	--spin->si_free_count;
    }
#ifdef SPELL_PRINTTREE
    if (n != NULL)
	n->wn_nr = ++spin->si_wordnode_nr;
#endif
    return n;
}

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

struct affentry_S
{
    affentry_T	*ae_next;	/* next affix with same name/number */
    char_u	*ae_chop;	/* text to chop off basic word (can be NULL) */
    char_u	*ae_add;	/* text to add to basic word (can be NULL) */
    char_u	*ae_flags;	/* flags on the affix (can be NULL) */
    char_u	*ae_cond;	/* condition (NULL for ".") */
    regprog_T	*ae_prog;	/* regexp program for ae_cond or NULL */
    char	ae_compforbid;	/* COMPOUNDFORBIDFLAG found */
    char	ae_comppermit;	/* COMPOUNDPERMITFLAG found */
}

typedef struct wordnode_S wordnode_T;


write_vim_spell(spellinfo_T *spin, char_u *fname)
{
    FILE	*fd;
    int		regionmask;
    int		round;
    wordnode_T	*tree;
    int		nodecount;
    int		i;
    int		l;
    garray_T	*gap;
    fromto_T	*ftp;
    char_u	*p;
    int		rr;
    int		retval = OK;
    size_t	fwv = 1;  /* collect return value of fwrite() to avoid
			     warnings from picky compiler */

    fd = mch_fopen((char *)fname, "w");
    if (fd == NULL)
    {
	EMSG2(_(e_notopen), fname);
	return FAIL;
    }

    /* <HEADER>: <fileID> <versionnr> */
							    /* <fileID> */
    fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd);
    if (fwv != (size_t)1)
	/* Catch first write error, don't try writing more. */
	goto theend;

    putc(VIMSPELLVERSION, fd);				    /* <versionnr> */

    /*
     * <SECTIONS>: <section> ... <sectionend>
     */

    /* SN_INFO: <infotext> */
    if (spin->si_info != NULL)
    {
	putc(SN_INFO, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	i = (int)STRLEN(spin->si_info);
	put_bytes(fd, (long_u)i, 4);			/* <sectionlen> */
	fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); /* <infotext> */
    }

    /* SN_REGION: <regionname> ...
     * Write the region names only if there is more than one. */
    if (spin->si_region_count > 1)
    {
	putc(SN_REGION, fd);				/* <sectionID> */
	putc(SNF_REQUIRED, fd);				/* <sectionflags> */
	l = spin->si_region_count * 2;
	put_bytes(fd, (long_u)l, 4);			/* <sectionlen> */
	fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd);
							/* <regionname> ... */
	regionmask = (1 << spin->si_region_count) - 1;
    }
    else
	regionmask = 0;

    /* SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars>
     *
     * The table with character flags and the table for case folding.
     * This makes sure the same characters are recognized as word characters
     * when generating an when using a spell file.
     * Skip this for ASCII, the table may conflict with the one used for
     * 'encoding'.
     * Also skip this for an .add.spl file, the main spell file must contain
     * the table (avoids that it conflicts).  File is shorter too.
     */
    if (!spin->si_ascii && !spin->si_add)
    {
	char_u	folchars[128 * 8];
	int	flags;

	putc(SN_CHARFLAGS, fd);				/* <sectionID> */
	putc(SNF_REQUIRED, fd);				/* <sectionflags> */

	/* Form the <folchars> string first, we need to know its length. */
	l = 0;
	for (i = 128; i < 256; ++i)
	{
#ifdef FEAT_MBYTE
	    if (has_mbyte)
		l += mb_char2bytes(spelltab.st_fold[i], folchars + l);
	    else
#endif
		folchars[l++] = spelltab.st_fold[i];
	}
	put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4);	/* <sectionlen> */

	fputc(128, fd);					/* <charflagslen> */
	for (i = 128; i < 256; ++i)
	{
	    flags = 0;
	    if (spelltab.st_isw[i])
		flags |= CF_WORD;
	    if (spelltab.st_isu[i])
		flags |= CF_UPPER;
	    fputc(flags, fd);				/* <charflags> */
	}

	put_bytes(fd, (long_u)l, 2);			/* <folcharslen> */
	fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); /* <folchars> */
    }

    /* SN_MIDWORD: <midword> */
    if (spin->si_midword != NULL)
    {
	putc(SN_MIDWORD, fd);				/* <sectionID> */
	putc(SNF_REQUIRED, fd);				/* <sectionflags> */

	i = (int)STRLEN(spin->si_midword);
	put_bytes(fd, (long_u)i, 4);			/* <sectionlen> */
	fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd);
							/* <midword> */
    }

    /* SN_PREFCOND: <prefcondcnt> <prefcond> ... */
    if (spin->si_prefcond.ga_len > 0)
    {
	putc(SN_PREFCOND, fd);				/* <sectionID> */
	putc(SNF_REQUIRED, fd);				/* <sectionflags> */

	l = write_spell_prefcond(NULL, &spin->si_prefcond);
	put_bytes(fd, (long_u)l, 4);			/* <sectionlen> */

	write_spell_prefcond(fd, &spin->si_prefcond);
    }

    /* SN_REP: <repcount> <rep> ...
     * SN_SAL: <salflags> <salcount> <sal> ...
     * SN_REPSAL: <repcount> <rep> ... */

    /* round 1: SN_REP section
     * round 2: SN_SAL section (unless SN_SOFO is used)
     * round 3: SN_REPSAL section */
    for (round = 1; round <= 3; ++round)
    {
	if (round == 1)
	    gap = &spin->si_rep;
	else if (round == 2)
	{
	    /* Don't write SN_SAL when using a SN_SOFO section */
	    if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
		continue;
	    gap = &spin->si_sal;
	}
	else
	    gap = &spin->si_repsal;

	/* Don't write the section if there are no items. */
	if (gap->ga_len == 0)
	    continue;

	/* Sort the REP/REPSAL items. */
	if (round != 2)
	    qsort(gap->ga_data, (size_t)gap->ga_len,
					       sizeof(fromto_T), rep_compare);

	i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL);
	putc(i, fd);					/* <sectionID> */

	/* This is for making suggestions, section is not required. */
	putc(0, fd);					/* <sectionflags> */

	/* Compute the length of what follows. */
	l = 2;	    /* count <repcount> or <salcount> */
	for (i = 0; i < gap->ga_len; ++i)
	{
	    ftp = &((fromto_T *)gap->ga_data)[i];
	    l += 1 + (int)STRLEN(ftp->ft_from);  /* count <*fromlen> and <*from> */
	    l += 1 + (int)STRLEN(ftp->ft_to);    /* count <*tolen> and <*to> */
	}
	if (round == 2)
	    ++l;	/* count <salflags> */
	put_bytes(fd, (long_u)l, 4);			/* <sectionlen> */

	if (round == 2)
	{
	    i = 0;
	    if (spin->si_followup)
		i |= SAL_F0LLOWUP;
	    if (spin->si_collapse)
		i |= SAL_COLLAPSE;
	    if (spin->si_rem_accents)
		i |= SAL_REM_ACCENTS;
	    putc(i, fd);			/* <salflags> */
	}

	put_bytes(fd, (long_u)gap->ga_len, 2);	/* <repcount> or <salcount> */
	for (i = 0; i < gap->ga_len; ++i)
	{
	    /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */
	    /* <sal> : <salfromlen> <salfrom> <saltolen> <salto> */
	    ftp = &((fromto_T *)gap->ga_data)[i];
	    for (rr = 1; rr <= 2; ++rr)
	    {
		p = rr == 1 ? ftp->ft_from : ftp->ft_to;
		l = (int)STRLEN(p);
		putc(l, fd);
		if (l > 0)
		    fwv &= fwrite(p, l, (size_t)1, fd);
	    }
	}

    }

    /* SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto>
     * This is for making suggestions, section is not required. */
    if (spin->si_sofofr != NULL && spin->si_sofoto != NULL)
    {
	putc(SN_SOFO, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	l = (int)STRLEN(spin->si_sofofr);
	put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4);
							/* <sectionlen> */

	put_bytes(fd, (long_u)l, 2);			/* <sofofromlen> */
	fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); /* <sofofrom> */

	l = (int)STRLEN(spin->si_sofoto);
	put_bytes(fd, (long_u)l, 2);			/* <sofotolen> */
	fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); /* <sofoto> */
    }

    /* SN_WORDS: <word> ...
     * This is for making suggestions, section is not required. */
    if (spin->si_commonwords.ht_used > 0)
    {
	putc(SN_WORDS, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	/* round 1: count the bytes
	 * round 2: write the bytes */
	for (round = 1; round <= 2; ++round)
	{
	    int		todo;
	    int		len = 0;
	    hashitem_T	*hi;

	    todo = (int)spin->si_commonwords.ht_used;
	    for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi)
		if (!HASHITEM_EMPTY(hi))
		{
		    l = (int)STRLEN(hi->hi_key) + 1;
		    len += l;
		    if (round == 2)			/* <word> */
			fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd);
		    --todo;
		}
	    if (round == 1)
		put_bytes(fd, (long_u)len, 4);		/* <sectionlen> */
	}
    }

    /* SN_MAP: <mapstr>
     * This is for making suggestions, section is not required. */
    if (spin->si_map.ga_len > 0)
    {
	putc(SN_MAP, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */
	l = spin->si_map.ga_len;
	put_bytes(fd, (long_u)l, 4);			/* <sectionlen> */
	fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd);
							/* <mapstr> */
    }

    /* SN_SUGFILE: <timestamp>
     * This is used to notify that a .sug file may be available and at the
     * same time allows for checking that a .sug file that is found matches
     * with this .spl file.  That's because the word numbers must be exactly
     * right. */
    if (!spin->si_nosugfile
	    && (spin->si_sal.ga_len > 0
		     || (spin->si_sofofr != NULL && spin->si_sofoto != NULL)))
    {
	putc(SN_SUGFILE, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */
	put_bytes(fd, (long_u)8, 4);			/* <sectionlen> */

	/* Set si_sugtime and write it to the file. */
	spin->si_sugtime = time(NULL);
	put_time(fd, spin->si_sugtime);			/* <timestamp> */
    }

    /* SN_NOSPLITSUGS: nothing
     * This is used to notify that no suggestions with word splits are to be
     * made. */
    if (spin->si_nosplitsugs)
    {
	putc(SN_NOSPLITSUGS, fd);			/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */
	put_bytes(fd, (long_u)0, 4);			/* <sectionlen> */
    }

    /* SN_NOCOMPUNDSUGS: nothing
     * This is used to notify that no suggestions with compounds are to be
     * made. */
    if (spin->si_nocompoundsugs)
    {
	putc(SN_NOCOMPOUNDSUGS, fd);			/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */
	put_bytes(fd, (long_u)0, 4);			/* <sectionlen> */
    }

    /* SN_COMPOUND: compound info.
     * We don't mark it required, when not supported all compound words will
     * be bad words. */
    if (spin->si_compflags != NULL)
    {
	putc(SN_COMPOUND, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	l = (int)STRLEN(spin->si_compflags);
	for (i = 0; i < spin->si_comppat.ga_len; ++i)
	    l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1;
	put_bytes(fd, (long_u)(l + 7), 4);		/* <sectionlen> */

	putc(spin->si_compmax, fd);			/* <compmax> */
	putc(spin->si_compminlen, fd);			/* <compminlen> */
	putc(spin->si_compsylmax, fd);			/* <compsylmax> */
	putc(0, fd);		/* for Vim 7.0b compatibility */
	putc(spin->si_compoptions, fd);			/* <compoptions> */
	put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2);
							/* <comppatcount> */
	for (i = 0; i < spin->si_comppat.ga_len; ++i)
	{
	    p = ((char_u **)(spin->si_comppat.ga_data))[i];
	    putc((int)STRLEN(p), fd);			/* <comppatlen> */
	    fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd);
							/* <comppattext> */
	}
							/* <compflags> */
	fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags),
							       (size_t)1, fd);
    }

    /* SN_NOBREAK: NOBREAK flag */
    if (spin->si_nobreak)
    {
	putc(SN_NOBREAK, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	/* It's empty, the presence of the section flags the feature. */
	put_bytes(fd, (long_u)0, 4);			/* <sectionlen> */
    }

    /* SN_SYLLABLE: syllable info.
     * We don't mark it required, when not supported syllables will not be
     * counted. */
    if (spin->si_syllable != NULL)
    {
	putc(SN_SYLLABLE, fd);				/* <sectionID> */
	putc(0, fd);					/* <sectionflags> */

	l = (int)STRLEN(spin->si_syllable);
	put_bytes(fd, (long_u)l, 4);			/* <sectionlen> */
	fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd);
							/* <syllable> */
    }

    /* end of <SECTIONS> */
    putc(SN_END, fd);					/* <sectionend> */

    /*
     * <LWORDTREE>  <KWORDTREE>  <PREFIXTREE>
     */
    spin->si_memtot = 0;
    for (round = 1; round <= 3; ++round)
    {
	if (round == 1)
	    tree = spin->si_foldroot->wn_sibling;
	else if (round == 2)
	    tree = spin->si_keeproot->wn_sibling;
	else
	    tree = spin->si_prefroot->wn_sibling;

	/* Clear the index and wnode fields in the tree. */
	clear_node(tree);

	/* Count the number of nodes.  Needed to be able to allocate the
	 * memory when reading the nodes.  Also fills in index for shared
	 * nodes. */
	nodecount = put_node(NULL, tree, 0, regionmask, round == 3);

	/* number of nodes in 4 bytes */
	put_bytes(fd, (long_u)nodecount, 4);	/* <nodecount> */
	spin->si_memtot += nodecount + nodecount * sizeof(int);

	/* Write the nodes. */
	(void)put_node(fd, tree, 0, regionmask, round == 3);
    }

    /* Write another byte to check for errors (file system full). */
    if (putc(0, fd) == EOF)
	retval = FAIL;
theend:
    if (fclose(fd) == EOF)
	retval = FAIL;

    if (fwv != (size_t)1)
	retval = FAIL;
    if (retval == FAIL)
	EMSG(_(e_write));

    return retval;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SAL_F0LLOWUP		1

#define SAL_COLLAPSE		2

#define SAL_REM_ACCENTS		4

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define VIMSPELLMAGICL 8

#define VIMSPELLVERSION 50

#define SN_REGION	0	/* <regionname> section */

#define SN_CHARFLAGS	1	/* charflags section */

#define SN_MIDWORD	2	/* <midword> section */

#define SN_PREFCOND	3	/* <prefcond> section */

#define SN_REP		4	/* REP items section */

#define SN_SAL		5	/* SAL items section */

#define SN_SOFO		6	/* soundfolding section */

#define SN_MAP		7	/* MAP items section */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SYLLABLE	9	/* syllable section */

#define SN_NOBREAK	10	/* NOBREAK section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_REPSAL	12	/* REPSAL items section */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SN_END		255	/* end of sections */

#define SNF_REQUIRED	1	/* <sectionflags>: required section */

#define CF_WORD		0x01

#define CF_UPPER	0x02

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);


get_pfxlist(
    afffile_T	*affile,
    char_u	*afflist,
    char_u	*store_afflist)
{
    char_u	*p;
    char_u	*prevp;
    int		cnt = 0;
    int		id;
    char_u	key[AH_KEY_LEN];
    hashitem_T	*hi;

    for (p = afflist; *p != NUL; )
    {
	prevp = p;
	if (get_affitem(affile->af_flagtype, &p) != 0)
	{
	    /* A flag is a postponed prefix flag if it appears in "af_pref"
	     * and it's ID is not zero. */
	    vim_strncpy(key, prevp, p - prevp);
	    hi = hash_find(&affile->af_pref, key);
	    if (!HASHITEM_EMPTY(hi))
	    {
		id = HI2AH(hi)->ah_newID;
		if (id != 0)
		    store_afflist[cnt++] = id;
	    }
	}
	if (affile->af_flagtype == AFT_NUM && *p == ',')
	    ++p;
    }

    store_afflist[cnt] = NUL;
    return cnt;
}


set_spell_finish(spelltab_T *new_st)
{
    int		i;

    if (did_set_spelltab)
    {
	/* check that it's the same table */
	for (i = 0; i < 256; ++i)
	{
	    if (spelltab.st_isw[i] != new_st->st_isw[i]
		    || spelltab.st_isu[i] != new_st->st_isu[i]
		    || spelltab.st_fold[i] != new_st->st_fold[i]
		    || spelltab.st_upper[i] != new_st->st_upper[i])
	    {
		EMSG(_("E763: Word characters differ between spell files"));
		return FAIL;
	    }
	}
    }
    else
    {
	/* copy the new spelltab into the one being used */
	spelltab = *new_st;
	did_set_spelltab = TRUE;
    }

    return OK;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static char *e_spell_trunc = N_("E758: Truncated spell file");


spell_message(spellinfo_T *spin, char_u *str)
{
    if (spin->si_verbose || p_verbose > 2)
    {
	if (!spin->si_verbose)
	    verbose_enter();
	MSG(str);
	out_flush();
	if (!spin->si_verbose)
	    verbose_leave();
    }
}

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

struct affentry_S
{
    affentry_T	*ae_next;	/* next affix with same name/number */
    char_u	*ae_chop;	/* text to chop off basic word (can be NULL) */
    char_u	*ae_add;	/* text to add to basic word (can be NULL) */
    char_u	*ae_flags;	/* flags on the affix (can be NULL) */
    char_u	*ae_cond;	/* condition (NULL for ".") */
    regprog_T	*ae_prog;	/* regexp program for ae_cond or NULL */
    char	ae_compforbid;	/* COMPOUNDFORBIDFLAG found */
    char	ae_comppermit;	/* COMPOUNDPERMITFLAG found */
}


sug_filltree(spellinfo_T *spin, slang_T *slang)
{
    char_u	*byts;
    idx_T	*idxs;
    int		depth;
    idx_T	arridx[MAXWLEN];
    int		curi[MAXWLEN];
    char_u	tword[MAXWLEN];
    char_u	tsalword[MAXWLEN];
    int		c;
    idx_T	n;
    unsigned	words_done = 0;
    int		wordcount[MAXWLEN];

    /* We use si_foldroot for the soundfolded trie. */
    spin->si_foldroot = wordtree_alloc(spin);
    if (spin->si_foldroot == NULL)
	return FAIL;

    /* let tree_add_word() know we're adding to the soundfolded tree */
    spin->si_sugtree = TRUE;

    /*
     * Go through the whole case-folded tree, soundfold each word and put it
     * in the trie.
     */
    byts = slang->sl_fbyts;
    idxs = slang->sl_fidxs;

    arridx[0] = 0;
    curi[0] = 1;
    wordcount[0] = 0;

    depth = 0;
    while (depth >= 0 && !got_int)
    {
	if (curi[depth] > byts[arridx[depth]])
	{
	    /* Done all bytes at this node, go up one level. */
	    idxs[arridx[depth]] = wordcount[depth];
	    if (depth > 0)
		wordcount[depth - 1] += wordcount[depth];

	    --depth;
	    line_breakcheck();
	}
	else
	{

	    /* Do one more byte at this node. */
	    n = arridx[depth] + curi[depth];
	    ++curi[depth];

	    c = byts[n];
	    if (c == 0)
	    {
		/* Sound-fold the word. */
		tword[depth] = NUL;
		spell_soundfold(slang, tword, TRUE, tsalword);

		/* We use the "flags" field for the MSB of the wordnr,
		 * "region" for the LSB of the wordnr.  */
		if (tree_add_word(spin, tsalword, spin->si_foldroot,
				words_done >> 16, words_done & 0xffff,
							   0) == FAIL)
		    return FAIL;

		++words_done;
		++wordcount[depth];

		/* Reset the block count each time to avoid compression
		 * kicking in. */
		spin->si_blocks_cnt = 0;

		/* Skip over any other NUL bytes (same word with different
		 * flags). */
		while (byts[n + 1] == 0)
		{
		    ++n;
		    ++curi[depth];
		}
	    }
	    else
	    {
		/* Normal char, go one level deeper. */
		tword[depth++] = c;
		arridx[depth] = idxs[n];
		curi[depth] = 1;
		wordcount[depth] = 0;
	    }
	}
    }

    smsg((char_u *)_("Total number of words: %d"), words_done);

    return OK;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


sug_filltable(
    spellinfo_T	*spin,
    wordnode_T	*node,
    int		startwordnr,
    garray_T	*gap)	    /* place to store line of numbers */
{
    wordnode_T	*p, *np;
    int		wordnr = startwordnr;
    int		nr;
    int		prev_nr;

    for (p = node; p != NULL; p = p->wn_sibling)
    {
	if (p->wn_byte == NUL)
	{
	    gap->ga_len = 0;
	    prev_nr = 0;
	    for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling)
	    {
		if (ga_grow(gap, 10) == FAIL)
		    return -1;

		nr = (np->wn_flags << 16) + (np->wn_region & 0xffff);
		/* Compute the offset from the previous nr and store the
		 * offset in a way that it takes a minimum number of bytes.
		 * It's a bit like utf-8, but without the need to mark
		 * following bytes. */
		nr -= prev_nr;
		prev_nr += nr;
		gap->ga_len += offset2bytes(nr,
					 (char_u *)gap->ga_data + gap->ga_len);
	    }

	    /* add the NUL byte */
	    ((char_u *)gap->ga_data)[gap->ga_len++] = NUL;

	    if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr,
				     gap->ga_data, gap->ga_len, TRUE) == FAIL)
		return -1;
	    ++wordnr;

	    /* Remove extra NUL entries, we no longer need them. We don't
	     * bother freeing the nodes, the won't be reused anyway. */
	    while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL)
		p->wn_sibling = p->wn_sibling->wn_sibling;

	    /* Clear the flags on the remaining NUL node, so that compression
	     * works a lot better. */
	    p->wn_flags = 0;
	    p->wn_region = 0;
	}
	else
	{
	    wordnr = sug_filltable(spin, p->wn_child, wordnr, gap);
	    if (wordnr == -1)
		return -1;
	}
    }
    return wordnr;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


node_compress(
    spellinfo_T	*spin,
    wordnode_T	*node,
    hashtab_T	*ht,
    int		*tot)	    /* total count of nodes before compressing,
			       incremented while going through the tree */
{
    wordnode_T	*np;
    wordnode_T	*tp;
    wordnode_T	*child;
    hash_T	hash;
    hashitem_T	*hi;
    int		len = 0;
    unsigned	nr, n;
    int		compressed = 0;

    /*
     * Go through the list of siblings.  Compress each child and then try
     * finding an identical child to replace it.
     * Note that with "child" we mean not just the node that is pointed to,
     * but the whole list of siblings of which the child node is the first.
     */
    for (np = node; np != NULL && !got_int; np = np->wn_sibling)
    {
	++len;
	if ((child = np->wn_child) != NULL)
	{
	    /* Compress the child first.  This fills hashkey. */
	    compressed += node_compress(spin, child, ht, tot);

	    /* Try to find an identical child. */
	    hash = hash_hash(child->wn_u1.hashkey);
	    hi = hash_lookup(ht, child->wn_u1.hashkey, hash);
	    if (!HASHITEM_EMPTY(hi))
	    {
		/* There are children we encountered before with a hash value
		 * identical to the current child.  Now check if there is one
		 * that is really identical. */
		for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next)
		    if (node_equal(child, tp))
		    {
			/* Found one!  Now use that child in place of the
			 * current one.  This means the current child and all
			 * its siblings is unlinked from the tree. */
			++tp->wn_refs;
			compressed += deref_wordnode(spin, child);
			np->wn_child = tp;
			break;
		    }
		if (tp == NULL)
		{
		    /* No other child with this hash value equals the child of
		     * the node, add it to the linked list after the first
		     * item. */
		    tp = HI2WN(hi);
		    child->wn_u2.next = tp->wn_u2.next;
		    tp->wn_u2.next = child;
		}
	    }
	    else
		/* No other child has this hash value, add it to the
		 * hashtable. */
		hash_add_item(ht, hi, child->wn_u1.hashkey, hash);
	}
    }
    *tot += len + 1;	/* add one for the node that stores the length */

    /*
     * Make a hash key for the node and its siblings, so that we can quickly
     * find a lookalike node.  This must be done after compressing the sibling
     * list, otherwise the hash key would become invalid by the compression.
     */
    node->wn_u1.hashkey[0] = len;
    nr = 0;
    for (np = node; np != NULL; np = np->wn_sibling)
    {
	if (np->wn_byte == NUL)
	    /* end node: use wn_flags, wn_region and wn_affixID */
	    n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16);
	else
	    /* byte node: use the byte value and the child pointer */
	    n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8));
	nr = nr * 101 + n;
    }

    /* Avoid NUL bytes, it terminates the hash key. */
    n = nr & 0xff;
    node->wn_u1.hashkey[1] = n == 0 ? 1 : n;
    n = (nr >> 8) & 0xff;
    node->wn_u1.hashkey[2] = n == 0 ? 1 : n;
    n = (nr >> 16) & 0xff;
    node->wn_u1.hashkey[3] = n == 0 ? 1 : n;
    n = (nr >> 24) & 0xff;
    node->wn_u1.hashkey[4] = n == 0 ? 1 : n;
    node->wn_u1.hashkey[5] = NUL;

    /* Check for CTRL-C pressed now and then. */
    fast_breakcheck();

    return compressed;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SN_END		255	/* end of sections */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


offset2bytes(int nr, char_u *buf)
{
    int	    rem;
    int	    b1, b2, b3, b4;

    /* Split the number in parts of base 255.  We need to avoid NUL bytes. */
    b1 = nr % 255 + 1;
    rem = nr / 255;
    b2 = rem % 255 + 1;
    rem = rem / 255;
    b3 = rem % 255 + 1;
    b4 = rem / 255 + 1;

    if (b4 > 1 || b3 > 0x1f)	/* 4 bytes */
    {
	buf[0] = 0xe0 + b4;
	buf[1] = b3;
	buf[2] = b2;
	buf[3] = b1;
	return 4;
    }
    if (b3 > 1 || b2 > 0x3f )	/* 3 bytes */
    {
	buf[0] = 0xc0 + b3;
	buf[1] = b2;
	buf[2] = b1;
	return 3;
    }
    if (b2 > 1 || b1 > 0x7f )	/* 2 bytes */
    {
	buf[0] = 0x80 + b2;
	buf[1] = b1;
	return 2;
    }
				/* 1 byte */
    buf[0] = b1;
    return 1;
}

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


aff_process_flags(afffile_T *affile, affentry_T *entry)
{
    char_u	*p;
    char_u	*prevp;
    unsigned	flag;

    if (entry->ae_flags != NULL
		&& (affile->af_compforbid != 0 || affile->af_comppermit != 0))
    {
	for (p = entry->ae_flags; *p != NUL; )
	{
	    prevp = p;
	    flag = get_affitem(affile->af_flagtype, &p);
	    if (flag == affile->af_comppermit || flag == affile->af_compforbid)
	    {
		STRMOVE(prevp, p);
		p = prevp;
		if (flag == affile->af_comppermit)
		    entry->ae_comppermit = TRUE;
		else
		    entry->ae_compforbid = TRUE;
	    }
	    if (affile->af_flagtype == AFT_NUM && *p == ',')
		++p;
	}
	if (*entry->ae_flags == NUL)
	    entry->ae_flags = NULL;	/* nothing left */
    }
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

#define AFT_NUM		3	/* flags are numbers, comma separated */

typedef struct affentry_S affentry_T;


mb_str2wide(char_u *s)
{
    int		*res;
    char_u	*p;
    int		i = 0;

    res = (int *)alloc(sizeof(int) * (mb_charlen(s) + 1));
    if (res != NULL)
    {
	for (p = s; *p != NUL; )
	    res[i++] = mb_ptr2char_adv(&p);
	res[i] = NUL;
    }
    return res;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;