xsltInitCtxtKey(xsltTransformContextPtr ctxt, xsltDocumentPtr idoc,
	        xsltKeyDefPtr keyDef)
{
    int i, len, k;
    xmlNodeSetPtr matchList = NULL, keylist;
    xmlXPathObjectPtr matchRes = NULL, useRes = NULL;
    xmlChar *str = NULL;
    xsltKeyTablePtr table;
    xmlNodePtr oldInst, cur;
    xmlNodePtr oldContextNode;
    xsltDocumentPtr oldDocInfo;
    int	oldXPPos, oldXPSize;
    xmlDocPtr oldXPDoc;
    int oldXPNsNr;
    xmlNsPtr *oldXPNamespaces;
    xmlXPathContextPtr xpctxt;

#ifdef KEY_INIT_DEBUG
fprintf(stderr, "xsltInitCtxtKey %s : %d\n", keyDef->name, ctxt->keyInitLevel);
#endif

    if ((keyDef->comp == NULL) || (keyDef->usecomp == NULL))
	return(-1);

    /*
     * Detect recursive keys
     */
    if (ctxt->keyInitLevel > ctxt->nbKeys) {
#ifdef WITH_XSLT_DEBUG_KEYS
	XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,
	           xsltGenericDebug(xsltGenericDebugContext,
		       "xsltInitCtxtKey: key definition of %s is recursive\n",
		       keyDef->name));
#endif
	xsltTransformError(ctxt, NULL, keyDef->inst,
	    "Key definition for %s is recursive\n", keyDef->name);
	ctxt->state = XSLT_STATE_STOPPED;
        return(-1);
    }
    ctxt->keyInitLevel++;

    xpctxt = ctxt->xpathCtxt;
    idoc->nbKeysComputed++;
    /*
    * Save context state.
    */
    oldInst = ctxt->inst;
    oldDocInfo = ctxt->document;
    oldContextNode = ctxt->node;

    oldXPDoc = xpctxt->doc;
    oldXPPos = xpctxt->proximityPosition;
    oldXPSize = xpctxt->contextSize;
    oldXPNsNr = xpctxt->nsNr;
    oldXPNamespaces = xpctxt->namespaces;

    /*
    * Set up contexts.
    */
    ctxt->document = idoc;
    ctxt->node = (xmlNodePtr) idoc->doc;
    ctxt->inst = keyDef->inst;

    xpctxt->doc = idoc->doc;
    xpctxt->node = (xmlNodePtr) idoc->doc;
    /* TODO : clarify the use of namespaces in keys evaluation */
    xpctxt->namespaces = keyDef->nsList;
    xpctxt->nsNr = keyDef->nsNr;

    /*
    * Evaluate the 'match' expression of the xsl:key.
    * TODO: The 'match' is a *pattern*.
    */
    matchRes = xmlXPathCompiledEval(keyDef->comp, xpctxt);
    if (matchRes == NULL) {

#ifdef WITH_XSLT_DEBUG_KEYS
	XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,xsltGenericDebug(xsltGenericDebugContext,
	     "xsltInitCtxtKey: %s evaluation failed\n", keyDef->match));
#endif
	xsltTransformError(ctxt, NULL, keyDef->inst,
	    "Failed to evaluate the 'match' expression.\n");
	ctxt->state = XSLT_STATE_STOPPED;
	goto error;
    } else {
	if (matchRes->type == XPATH_NODESET) {
	    matchList = matchRes->nodesetval;

#ifdef WITH_XSLT_DEBUG_KEYS
	    if (matchList != NULL)
		XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,xsltGenericDebug(xsltGenericDebugContext,
		     "xsltInitCtxtKey: %s evaluates to %d nodes\n",
				 keyDef->match, matchList->nodeNr));
#endif
	} else {
	    /*
	    * Is not a node set, but must be.
	    */
#ifdef WITH_XSLT_DEBUG_KEYS
	    XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,xsltGenericDebug(xsltGenericDebugContext,
		 "xsltInitCtxtKey: %s is not a node set\n", keyDef->match));
#endif
	    xsltTransformError(ctxt, NULL, keyDef->inst,
		"The 'match' expression did not evaluate to a node set.\n");
	    ctxt->state = XSLT_STATE_STOPPED;
	    goto error;
	}
    }
    if ((matchList == NULL) || (matchList->nodeNr <= 0))
	goto exit;

    /**
     * Multiple key definitions for the same name are allowed, so
     * we must check if the key is already present for this doc
     */
    table = (xsltKeyTablePtr) idoc->keys;
    while (table != NULL) {
        if (xmlStrEqual(table->name, keyDef->name) &&
	    (((keyDef->nameURI == NULL) && (table->nameURI == NULL)) ||
	     ((keyDef->nameURI != NULL) && (table->nameURI != NULL) &&
	      (xmlStrEqual(table->nameURI, keyDef->nameURI)))))
	    break;
	table = table->next;
    }
    /**
     * If the key was not previously defined, create it now and
     * chain it to the list of keys for the doc
     */
    if (table == NULL) {
        table = xsltNewKeyTable(keyDef->name, keyDef->nameURI);
        if (table == NULL)
	    goto error;
        table->next = idoc->keys;
        idoc->keys = table;
    }

    /*
    * SPEC XSLT 1.0 (XSLT 2.0 does not clarify the context size!)
    * "...the use attribute of the xsl:key element is evaluated with x as
    "  the current node and with a node list containing just x as the
    *  current node list"
    */
    xpctxt->contextSize = 1;
    xpctxt->proximityPosition = 1;

    for (i = 0; i < matchList->nodeNr; i++) {
	cur = matchList->nodeTab[i];
	if (! IS_XSLT_REAL_NODE(cur))
	    continue;
	xpctxt->node = cur;
	/*
	* Process the 'use' of the xsl:key.
	* SPEC XSLT 1.0:
	* "The use attribute is an expression specifying the values of
	*  the key; the expression is evaluated once for each node that
	*  matches the pattern."
	*/
	if (useRes != NULL)
	    xmlXPathFreeObject(useRes);
	useRes = xmlXPathCompiledEval(keyDef->usecomp, xpctxt);
	if (useRes == NULL) {
	    xsltTransformError(ctxt, NULL, keyDef->inst,
		"Failed to evaluate the 'use' expression.\n");
	    ctxt->state = XSLT_STATE_STOPPED;
	    break;
	}
	if (useRes->type == XPATH_NODESET) {
	    if ((useRes->nodesetval != NULL) &&
		(useRes->nodesetval->nodeNr != 0))
	    {
		len = useRes->nodesetval->nodeNr;
		str = xmlXPathCastNodeToString(useRes->nodesetval->nodeTab[0]);
	    } else {
		continue;
	    }
	} else {
	    len = 1;
	    if (useRes->type == XPATH_STRING) {
		/*
		* Consume the string value.
		*/
		str = useRes->stringval;
		useRes->stringval = NULL;
	    } else {
		str = xmlXPathCastToString(useRes);
	    }
	}
	/*
	* Process all strings.
	*/
	k = 0;
	while (1) {
	    if (str == NULL)
		goto next_string;

#ifdef WITH_XSLT_DEBUG_KEYS
	    XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,xsltGenericDebug(xsltGenericDebugContext,
		"xsl:key : node associated to ('%s', '%s')\n", keyDef->name, str));
#endif

	    keylist = xmlHashLookup(table->keys, str);
	    if (keylist == NULL) {
		keylist = xmlXPathNodeSetCreate(cur);
		if (keylist == NULL)
		    goto error;
		xmlHashAddEntry(table->keys, str, keylist);
	    } else {
		/*
		* TODO: How do we know if this function failed?
		*/
		xmlXPathNodeSetAdd(keylist, cur);
	    }
	    switch (cur->type) {
		case XML_ELEMENT_NODE:
		case XML_TEXT_NODE:
		case XML_CDATA_SECTION_NODE:
		case XML_PI_NODE:
		case XML_COMMENT_NODE:
		    cur->psvi = keyDef;
		    break;
		case XML_ATTRIBUTE_NODE:
		    ((xmlAttrPtr) cur)->psvi = keyDef;
		    break;
		case XML_DOCUMENT_NODE:
		case XML_HTML_DOCUMENT_NODE:
		    ((xmlDocPtr) cur)->psvi = keyDef;
		    break;
		default:
		    break;
	    }
	    xmlFree(str);
	    str = NULL;

next_string:
	    k++;
	    if (k >= len)
		break;
	    str = xmlXPathCastNodeToString(useRes->nodesetval->nodeTab[k]);
	}
    }

exit:
error:
    ctxt->keyInitLevel--;
    /*
    * Restore context state.
    */
    xpctxt->doc = oldXPDoc;
    xpctxt->nsNr = oldXPNsNr;
    xpctxt->namespaces = oldXPNamespaces;
    xpctxt->proximityPosition = oldXPPos;
    xpctxt->contextSize = oldXPSize;

    ctxt->node = oldContextNode;
    ctxt->document = oldDocInfo;
    ctxt->inst = oldInst;

    if (str)
	xmlFree(str);
    if (useRes != NULL)
	xmlXPathFreeObject(useRes);
    if (matchRes != NULL)
	xmlXPathFreeObject(matchRes);
    return(0);
}


xsltInitDocKeyTable(xsltTransformContextPtr ctxt, const xmlChar *name,
                    const xmlChar *nameURI)
{
    xsltStylesheetPtr style;
    xsltKeyDefPtr keyd = NULL;
    int found = 0;

#ifdef KEY_INIT_DEBUG
fprintf(stderr, "xsltInitDocKeyTable %s\n", name);
#endif

    style = ctxt->style;
    while (style != NULL) {
	keyd = (xsltKeyDefPtr) style->keys;
	while (keyd != NULL) {
	    if (((keyd->nameURI != NULL) ==
		 (nameURI != NULL)) &&
		xmlStrEqual(keyd->name, name) &&
		xmlStrEqual(keyd->nameURI, nameURI))
	    {
		xsltInitCtxtKey(ctxt, ctxt->document, keyd);
		if (ctxt->document->nbKeysComputed == ctxt->nbKeys)
		    return(0);
		found = 1;
	    }
	    keyd = keyd->next;
	}
	style = xsltNextImport(style);
    }
    if (found == 0) {
#ifdef WITH_XSLT_DEBUG_KEYS
	XSLT_TRACE(ctxt,XSLT_TRACE_KEYS,xsltGenericDebug(xsltGenericDebugContext,
	     "xsltInitDocKeyTable: did not found %s\n", name));
#endif
	xsltTransformError(ctxt, NULL, keyd? keyd->inst : NULL,
	    "Failed to find key definition for %s\n", name);
	ctxt->state = XSLT_STATE_STOPPED;
        return(-1);
    }
#ifdef KEY_INIT_DEBUG
fprintf(stderr, "xsltInitDocKeyTable %s done\n", name);
#endif
    return(0);
}

Fix crash with empty xsl:key/@match attribute

See https://bugzilla.gnome.org/show_bug.cgi?id=685328

Also improve some xsl:key error messages.

#define WITH_XSLT_DEBUG_KEYS

static int
xsltInitDocKeyTable(xsltTransformContextPtr ctxt, const xmlChar *name,
                    const xmlChar *nameURI);


xsltInitAllDocKeys(xsltTransformContextPtr ctxt)
{
    xsltStylesheetPtr style;
    xsltKeyDefPtr keyd;
    xsltKeyTablePtr table;

    if (ctxt == NULL)
	return(-1);

#ifdef KEY_INIT_DEBUG
fprintf(stderr, "xsltInitAllDocKeys %d %d\n",
        ctxt->document->nbKeysComputed, ctxt->nbKeys);
#endif

    if (ctxt->document->nbKeysComputed == ctxt->nbKeys)
	return(0);

    /*
    * TODO: This could be further optimized
    */
    style = ctxt->style;
    while (style) {
	keyd = (xsltKeyDefPtr) style->keys;
	while (keyd != NULL) {
#ifdef KEY_INIT_DEBUG
fprintf(stderr, "Init key %s\n", keyd->name);
#endif
	    /*
	    * Check if keys with this QName have been already
	    * computed.
	    */
	    table = (xsltKeyTablePtr) ctxt->document->keys;
	    while (table) {
		if (((keyd->nameURI != NULL) == (table->nameURI != NULL)) &&
		    xmlStrEqual(keyd->name, table->name) &&
		    xmlStrEqual(keyd->nameURI, table->nameURI))
		{
		    break;
		}
		table = table->next;
	    }
	    if (table == NULL) {
		/*
		* Keys with this QName have not been yet computed.
		*/
		xsltInitDocKeyTable(ctxt, keyd->name, keyd->nameURI);
	    }
	    keyd = keyd->next;
	}
	style = xsltNextImport(style);
    }
#ifdef KEY_INIT_DEBUG
fprintf(stderr, "xsltInitAllDocKeys: done\n");
#endif
    return(0);
}

static int
xsltInitDocKeyTable(xsltTransformContextPtr ctxt, const xmlChar *name,
                    const xmlChar *nameURI);


xsltAddKey(xsltStylesheetPtr style, const xmlChar *name,
	   const xmlChar *nameURI, const xmlChar *match,
	   const xmlChar *use, xmlNodePtr inst) {
    xsltKeyDefPtr key;
    xmlChar *pattern = NULL;
    int current, end, start, i = 0;

    if ((style == NULL) || (name == NULL) || (match == NULL) || (use == NULL))
	return(-1);

#ifdef WITH_XSLT_DEBUG_KEYS
    xsltGenericDebug(xsltGenericDebugContext,
	"Add key %s, match %s, use %s\n", name, match, use);
#endif

    key = xsltNewKeyDef(name, nameURI);
    key->match = xmlStrdup(match);
    key->use = xmlStrdup(use);
    key->inst = inst;
    key->nsList = xmlGetNsList(inst->doc, inst);
    if (key->nsList != NULL) {
        while (key->nsList[i] != NULL)
	    i++;
    }
    key->nsNr = i;

    /*
     * Split the | and register it as as many keys
     */
    current = end = 0;
    while (match[current] != 0) {
	start = current;
	while (IS_BLANK_CH(match[current]))
	    current++;
	end = current;
	while ((match[end] != 0) && (match[end] != '|')) {
	    if (match[end] == '[') {
	        end = skipPredicate(match, end);
		if (end <= 0) {
		    xsltTransformError(NULL, style, inst,
		        "xsl:key : 'match' pattern is malformed: %s",
		        key->match);
		    if (style != NULL) style->errors++;
		    goto error;
		}
	    } else
		end++;
	}
	if (current == end) {
	    xsltTransformError(NULL, style, inst,
			       "xsl:key : 'match' pattern is empty\n");
	    if (style != NULL) style->errors++;
	    goto error;
	}
	if (match[start] != '/') {
	    pattern = xmlStrcat(pattern, (xmlChar *)"//");
	    if (pattern == NULL) {
		if (style != NULL) style->errors++;
		goto error;
	    }
	}
	pattern = xmlStrncat(pattern, &match[start], end - start);
	if (pattern == NULL) {
	    if (style != NULL) style->errors++;
	    goto error;
	}

	if (match[end] == '|') {
	    pattern = xmlStrcat(pattern, (xmlChar *)"|");
	    end++;
	}
	current = end;
    }
    if (pattern == NULL) {
        xsltTransformError(NULL, style, inst,
                           "xsl:key : 'match' pattern is empty\n");
        if (style != NULL) style->errors++;
        goto error;
    }
#ifdef WITH_XSLT_DEBUG_KEYS
    xsltGenericDebug(xsltGenericDebugContext,
	"   resulting pattern %s\n", pattern);
#endif
    /*
    * XSLT-1: "It is an error for the value of either the use
    *  attribute or the match attribute to contain a
    *  VariableReference."
    * TODO: We should report a variable-reference at compile-time.
    *   Maybe a search for "$", if it occurs outside of quotation
    *   marks, could be sufficient.
    */
#ifdef XML_XPATH_NOVAR
    key->comp = xsltXPathCompileFlags(style, pattern, XML_XPATH_NOVAR);
#else
    key->comp = xsltXPathCompile(style, pattern);
#endif
    if (key->comp == NULL) {
	xsltTransformError(NULL, style, inst,
		"xsl:key : 'match' pattern compilation failed '%s'\n",
		         pattern);
	if (style != NULL) style->errors++;
    }
#ifdef XML_XPATH_NOVAR
    key->usecomp = xsltXPathCompileFlags(style, use, XML_XPATH_NOVAR);
#else
    key->usecomp = xsltXPathCompile(style, use);
#endif
    if (key->usecomp == NULL) {
	xsltTransformError(NULL, style, inst,
		"xsl:key : 'use' expression compilation failed '%s'\n",
		         use);
	if (style != NULL) style->errors++;
    }

    /*
     * Sometimes the stylesheet writer use the order to ease the
     * resolution of keys when they are dependant, keep the provided
     * order so add the new one at the end.
     */
    if (style->keys == NULL) {
	style->keys = key;
    } else {
        xsltKeyDefPtr prev = style->keys;

	while (prev->next != NULL)
	    prev = prev->next;

	prev->next = key;
    }
    key->next = NULL;

error:
    if (pattern != NULL)
	xmlFree(pattern);
    return(0);
}


xsltFreeKeyTableList(xsltKeyTablePtr keyt) {
    xsltKeyTablePtr cur;

    while (keyt != NULL) {
	cur = keyt;
	keyt = keyt->next;
	xsltFreeKeyTable(cur);
    }
}

#define IN_LIBXSLT

#define WITH_XSLT_DEBUG_KEYS

static int
xsltInitDocKeyTable(xsltTransformContextPtr ctxt, const xmlChar *name,
                    const xmlChar *nameURI);


xsltNewKeyDef(const xmlChar *name, const xmlChar *nameURI) {
    xsltKeyDefPtr cur;

    cur = (xsltKeyDefPtr) xmlMalloc(sizeof(xsltKeyDef));
    if (cur == NULL) {
	xsltTransformError(NULL, NULL, NULL,
		"xsltNewKeyDef : malloc failed\n");
	return(NULL);
    }
    memset(cur, 0, sizeof(xsltKeyDef));
    if (name != NULL)
	cur->name = xmlStrdup(name);
    if (nameURI != NULL)
	cur->nameURI = xmlStrdup(nameURI);
    cur->nsList = NULL;
    return(cur);
}


xsltGetKey(xsltTransformContextPtr ctxt, const xmlChar *name,
	   const xmlChar *nameURI, const xmlChar *value) {
    xmlNodeSetPtr ret;
    xsltKeyTablePtr table;
    int init_table = 0;

    if ((ctxt == NULL) || (name == NULL) || (value == NULL) ||
	(ctxt->document == NULL))
	return(NULL);

#ifdef WITH_XSLT_DEBUG_KEYS
    xsltGenericDebug(xsltGenericDebugContext,
	"Get key %s, value %s\n", name, value);
#endif

    /*
     * keys are computed only on-demand on first key access for a document
     */
    if ((ctxt->document->nbKeysComputed < ctxt->nbKeys) &&
        (ctxt->keyInitLevel == 0)) {
        /*
	 * If non-recursive behaviour, just try to initialize all keys
	 */
	if (xsltInitAllDocKeys(ctxt))
	    return(NULL);
    }

retry:
    table = (xsltKeyTablePtr) ctxt->document->keys;
    while (table != NULL) {
	if (((nameURI != NULL) == (table->nameURI != NULL)) &&
	    xmlStrEqual(table->name, name) &&
	    xmlStrEqual(table->nameURI, nameURI))
	{
	    ret = (xmlNodeSetPtr)xmlHashLookup(table->keys, value);
	    return(ret);
	}
	table = table->next;
    }

    if ((ctxt->keyInitLevel != 0) && (init_table == 0)) {
        /*
	 * Apparently one key is recursive and this one is needed,
	 * initialize just it, that time and retry
	 */
        xsltInitDocKeyTable(ctxt, name, nameURI);
	init_table = 1;
	goto retry;
    }

    return(NULL);
}


xsltFreeKeyDef(xsltKeyDefPtr keyd) {
    if (keyd == NULL)
	return;
    if (keyd->comp != NULL)
	xmlXPathFreeCompExpr(keyd->comp);
    if (keyd->usecomp != NULL)
	xmlXPathFreeCompExpr(keyd->usecomp);
    if (keyd->name != NULL)
	xmlFree(keyd->name);
    if (keyd->nameURI != NULL)
	xmlFree(keyd->nameURI);
    if (keyd->match != NULL)
	xmlFree(keyd->match);
    if (keyd->use != NULL)
	xmlFree(keyd->use);
    if (keyd->nsList != NULL)
        xmlFree(keyd->nsList);
    memset(keyd, -1, sizeof(xsltKeyDef));
    xmlFree(keyd);
}


void *zcalloc_usable(size_t size, size_t *usable) {
    void *ptr = calloc(1, size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = *usable = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}

Fix wrong zmalloc_size() assumption. (#7963)

When using a system with no malloc_usable_size(), zmalloc_size() assumed
that the heap allocator always returns blocks that are long-padded.

This may not always be the case, and will result with zmalloc_size()
returning a size that is bigger than allocated. At least in one case
this leads to out of bound write, process crash and a potential security
vulnerability.

Effectively this does not affect the vast majority of users, who use
jemalloc or glibc.

This problem along with a (different) fix was reported by Drew DeVault.

#define PREFIX_SIZE (0)

#define PREFIX_SIZE (sizeof(long long))

#define PREFIX_SIZE (sizeof(size_t))

#define malloc(size) tc_malloc(size)

#define calloc(count,size) tc_calloc(count,size)

#define realloc(ptr,size) tc_realloc(ptr,size)

#define free(ptr) tc_free(ptr)

#define update_zmalloc_stat_alloc(__n) atomicIncr(used_memory,(__n))

static redisAtomic size_t used_memory = 0;

static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;

#define UNUSED(x) ((void)(x))


void *zrealloc_usable(void *ptr, size_t size, size_t *usable) {
#ifndef HAVE_MALLOC_SIZE
    void *realptr;
#endif
    size_t oldsize;
    void *newptr;

    if (size == 0 && ptr != NULL) {
        zfree(ptr);
        *usable = 0;
        return NULL;
    }
    if (ptr == NULL) return zmalloc_usable(size, usable);
#ifdef HAVE_MALLOC_SIZE
    oldsize = zmalloc_size(ptr);
    newptr = realloc(ptr,size);
    if (!newptr) zmalloc_oom_handler(size);

    update_zmalloc_stat_free(oldsize);
    update_zmalloc_stat_alloc(*usable = zmalloc_size(newptr));
    return newptr;
#else
    realptr = (char*)ptr-PREFIX_SIZE;
    oldsize = *((size_t*)realptr);
    newptr = realloc(realptr,size+PREFIX_SIZE);
    if (!newptr) zmalloc_oom_handler(size);

    *((size_t*)newptr) = *usable = size;
    update_zmalloc_stat_free(oldsize);
    update_zmalloc_stat_alloc(size);
    return (char*)newptr+PREFIX_SIZE;
#endif
}

#define PREFIX_SIZE (0)

#define PREFIX_SIZE (sizeof(long long))

#define PREFIX_SIZE (sizeof(size_t))

#define malloc(size) tc_malloc(size)

#define calloc(count,size) tc_calloc(count,size)

#define realloc(ptr,size) tc_realloc(ptr,size)

#define free(ptr) tc_free(ptr)

#define update_zmalloc_stat_alloc(__n) atomicIncr(used_memory,(__n))

#define update_zmalloc_stat_free(__n) atomicDecr(used_memory,(__n))

static redisAtomic size_t used_memory = 0;

static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;

#define UNUSED(x) ((void)(x))


void *zmalloc_usable(size_t size, size_t *usable) {
    void *ptr = malloc(size+PREFIX_SIZE);

    if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
    update_zmalloc_stat_alloc(*usable = zmalloc_size(ptr));
    return ptr;
#else
    *((size_t*)ptr) = *usable = size;
    update_zmalloc_stat_alloc(size+PREFIX_SIZE);
    return (char*)ptr+PREFIX_SIZE;
#endif
}


static bool l2cap_check_enc_key_size(struct hci_conn *hcon)
{
	/* The minimum encryption key size needs to be enforced by the
	 * host stack before establishing any L2CAP connections. The
	 * specification in theory allows a minimum of 1, but to align
	 * BR/EDR and LE transports, a minimum of 7 is chosen.
	 *
	 * This check might also be called for unencrypted connections
	 * that have no key size requirements. Ensure that the link is
	 * actually encrypted before enforcing a key size.
	 */
	return (!test_bit(HCI_CONN_ENCRYPT, &hcon->flags) ||
		hcon->enc_key_size >= HCI_MIN_ENC_KEY_SIZE);
}

Bluetooth: Fix faulty expression for minimum encryption key size check

Fix minimum encryption key size check so that HCI_MIN_ENC_KEY_SIZE is
also allowed as stated in the comment.

This bug caused connection problems with devices having maximum
encryption key size of 7 octets (56-bit).

Fixes: 693cd8ce3f88 ("Bluetooth: Fix regression with minimum encryption key size alignment")
Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=203997
Signed-off-by: Matias Karhumaa <matias.karhumaa@gmail.com>
Cc: stable@vger.kernel.org
Signed-off-by: Marcel Holtmann <marcel@holtmann.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

bool disable_ertm;

static u32 l2cap_feat_mask = L2CAP_FEAT_FIXED_CHAN | L2CAP_FEAT_UCD;

static LIST_HEAD(chan_list);

static DEFINE_RWLOCK(chan_list_lock);

static struct sk_buff *l2cap_build_cmd(struct l2cap_conn *conn,
				       u8 code, u8 ident, u16 dlen, void *data);

static void l2cap_send_cmd(struct l2cap_conn *conn, u8 ident, u8 code, u16 len,
			   void *data);

static int l2cap_build_conf_req(struct l2cap_chan *chan, void *data, size_t data_size);

static void l2cap_send_disconn_req(struct l2cap_chan *chan, int err);

static void l2cap_tx(struct l2cap_chan *chan, struct l2cap_ctrl *control,
		     struct sk_buff_head *skbs, u8 event);

static struct hci_cb l2cap_cb = {
	.name		= "L2CAP",
	.connect_cfm	= l2cap_connect_cfm,
	.disconn_cfm	= l2cap_disconn_cfm,
	.security_cfm	= l2cap_security_cfm,
};

static struct dentry *l2cap_debugfs;


read_region_section(FILE *fd, slang_T *lp, int len)
{
    int		i;

    if (len > 16)
	return SP_FORMERROR;
    for (i = 0; i < len; ++i)
	lp->sl_regions[i] = getc(fd);			/* <regionname> */
    lp->sl_regions[len] = NUL;
    return 0;
}

patch 8.0.0322: possible overflow with corrupted spell file

Problem:    Possible overflow with spell file where the tree length is
            corrupted.
Solution:   Check for an invalid length (suggested by shqking)

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_REGION	0	/* <regionname> section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);


read_words_section(FILE *fd, slang_T *lp, int len)
{
    int		done = 0;
    int		i;
    int		c;
    char_u	word[MAXWLEN];

    while (done < len)
    {
	/* Read one word at a time. */
	for (i = 0; ; ++i)
	{
	    c = getc(fd);
	    if (c == EOF)
		return SP_TRUNCERROR;
	    word[i] = c;
	    if (word[i] == NUL)
		break;
	    if (i == MAXWLEN - 1)
		return SP_FORMERROR;
	}

	/* Init the count to 10. */
	count_common_word(lp, word, -1, 10);
	done += i + 1;
    }
    return 0;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *msg_compressing = N_("Compressing word tree...");


add_fromto(
    spellinfo_T	*spin,
    garray_T	*gap,
    char_u	*from,
    char_u	*to)
{
    fromto_T	*ftp;
    char_u	word[MAXWLEN];

    if (ga_grow(gap, 1) == OK)
    {
	ftp = ((fromto_T *)gap->ga_data) + gap->ga_len;
	(void)spell_casefold(from, (int)STRLEN(from), word, MAXWLEN);
	ftp->ft_from = getroom_save(spin, word);
	(void)spell_casefold(to, (int)STRLEN(to), word, MAXWLEN);
	ftp->ft_to = getroom_save(spin, word);
	++gap->ga_len;
    }
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *msg_compressing = N_("Compressing word tree...");


spell_clear_flags(wordnode_T *node)
{
    wordnode_T	*np;

    for (np = node; np != NULL; np = np->wn_sibling)
    {
	np->wn_u1.index = FALSE;
	spell_clear_flags(np->wn_child);
    }
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

struct affentry_S
{
    affentry_T	*ae_next;	/* next affix with same name/number */
    char_u	*ae_chop;	/* text to chop off basic word (can be NULL) */
    char_u	*ae_add;	/* text to add to basic word (can be NULL) */
    char_u	*ae_flags;	/* flags on the affix (can be NULL) */
    char_u	*ae_cond;	/* condition (NULL for ".") */
    regprog_T	*ae_prog;	/* regexp program for ae_cond or NULL */
    char	ae_compforbid;	/* COMPOUNDFORBIDFLAG found */
    char	ae_comppermit;	/* COMPOUNDPERMITFLAG found */
}

typedef struct affheader_S
{
    char_u	ah_key[AH_KEY_LEN]; /* key for hashtab == name of affix */
    unsigned	ah_flag;	/* affix name as number, uses "af_flagtype" */
    int		ah_newID;	/* prefix ID after renumbering; 0 if not used */
    int		ah_combine;	/* suffix may combine with prefix */
    int		ah_follows;	/* another affix block should be following */
    affentry_T	*ah_first;	/* first affix entry */
} affheader_T;


ex_mkspell(exarg_T *eap)
{
    int		fcount;
    char_u	**fnames;
    char_u	*arg = eap->arg;
    int		ascii = FALSE;

    if (STRNCMP(arg, "-ascii", 6) == 0)
    {
	ascii = TRUE;
	arg = skipwhite(arg + 6);
    }

    /* Expand all the remaining arguments (e.g., $VIMRUNTIME). */
    if (get_arglist_exp(arg, &fcount, &fnames, FALSE) == OK)
    {
	mkspell(fcount, fnames, ascii, eap->forceit, FALSE);
	FreeWild(fcount, fnames);
    }
}

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


wordtree_compress(spellinfo_T *spin, wordnode_T *root)
{
    hashtab_T	    ht;
    int		    n;
    int		    tot = 0;
    int		    perc;

    /* Skip the root itself, it's not actually used.  The first sibling is the
     * start of the tree. */
    if (root->wn_sibling != NULL)
    {
	hash_init(&ht);
	n = node_compress(spin, root->wn_sibling, &ht, &tot);

#ifndef SPELL_PRINTTREE
	if (spin->si_verbose || p_verbose > 2)
#endif
	{
	    if (tot > 1000000)
		perc = (tot - n) / (tot / 100);
	    else if (tot == 0)
		perc = 0;
	    else
		perc = (tot - n) * 100 / tot;
	    vim_snprintf((char *)IObuff, IOSIZE,
			  _("Compressed %d of %d nodes; %d (%d%%) remaining"),
						       n, tot, tot - n, perc);
	    spell_message(spin, IObuff);
	}
#ifdef SPELL_PRINTTREE
	spell_print_tree(root->wn_sibling);
#endif
	hash_clear(&ht);
    }
}

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");


tree_count_words(char_u *byts, idx_T *idxs)
{
    int		depth;
    idx_T	arridx[MAXWLEN];
    int		curi[MAXWLEN];
    int		c;
    idx_T	n;
    int		wordcount[MAXWLEN];

    arridx[0] = 0;
    curi[0] = 1;
    wordcount[0] = 0;
    depth = 0;
    while (depth >= 0 && !got_int)
    {
	if (curi[depth] > byts[arridx[depth]])
	{
	    /* Done all bytes at this node, go up one level. */
	    idxs[arridx[depth]] = wordcount[depth];
	    if (depth > 0)
		wordcount[depth - 1] += wordcount[depth];

	    --depth;
	    fast_breakcheck();
	}
	else
	{
	    /* Do one more byte at this node. */
	    n = arridx[depth] + curi[depth];
	    ++curi[depth];

	    c = byts[n];
	    if (c == 0)
	    {
		/* End of word, count it. */
		++wordcount[depth];

		/* Skip over any other NUL bytes (same word with different
		 * flags). */
		while (byts[n + 1] == 0)
		{
		    ++n;
		    ++curi[depth];
		}
	    }
	    else
	    {
		/* Normal char, go one level deeper to count the words. */
		++depth;
		arridx[depth] = idxs[n];
		curi[depth] = 1;
		wordcount[depth] = 0;
	    }
	}
    }
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_COMPOUND	8	/* compound words section */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


sug_maketable(spellinfo_T *spin)
{
    garray_T	ga;
    int		res = OK;

    /* Allocate a buffer, open a memline for it and create the swap file
     * (uses a temp file, not a .swp file). */
    spin->si_spellbuf = open_spellbuf();
    if (spin->si_spellbuf == NULL)
	return FAIL;

    /* Use a buffer to store the line info, avoids allocating many small
     * pieces of memory. */
    ga_init2(&ga, 1, 100);

    /* recursively go through the tree */
    if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1)
	res = FAIL;

    ga_clear(&ga);
    return res;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


spell_print_node(wordnode_T *node, int depth)
{
    if (node->wn_u1.index)
    {
	/* Done this node before, print the reference. */
	PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0);
	PRINTSOME(line2, depth, "    ", 0, 0);
	PRINTSOME(line3, depth, "    ", 0, 0);
	msg((char_u *)line1);
	msg((char_u *)line2);
	msg((char_u *)line3);
    }
    else
    {
	node->wn_u1.index = TRUE;

	if (node->wn_byte != NUL)
	{
	    if (node->wn_child != NULL)
		PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0);
	    else
		/* Cannot happen? */
		PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0);
	}
	else
	    PRINTSOME(line1, depth, " $    ", 0, 0);

	PRINTSOME(line2, depth, "%d/%d    ", node->wn_nr, node->wn_refs);

	if (node->wn_sibling != NULL)
	    PRINTSOME(line3, depth, " |    ", 0, 0);
	else
	    PRINTSOME(line3, depth, "      ", 0, 0);

	if (node->wn_byte == NUL)
	{
	    msg((char_u *)line1);
	    msg((char_u *)line2);
	    msg((char_u *)line3);
	}

	/* do the children */
	if (node->wn_byte != NUL && node->wn_child != NULL)
	    spell_print_node(node->wn_child, depth + 1);

	/* do the siblings */
	if (node->wn_sibling != NULL)
	{
	    /* get rid of all parent details except | */
	    STRCPY(line1, line3);
	    STRCPY(line2, line3);
	    spell_print_node(node->wn_sibling, depth);
	}
    }
}

#define BY_INDEX	1	/* child is shared, index follows */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


ex_spell(exarg_T *eap)
{
    spell_add_word(eap->arg, (int)STRLEN(eap->arg), eap->cmdidx == CMD_spellwrong,
				   eap->forceit ? 0 : (int)eap->line2,
				   eap->cmdidx == CMD_spellundo);
}

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

typedef struct affheader_S
{
    char_u	ah_key[AH_KEY_LEN]; /* key for hashtab == name of affix */
    unsigned	ah_flag;	/* affix name as number, uses "af_flagtype" */
    int		ah_newID;	/* prefix ID after renumbering; 0 if not used */
    int		ah_combine;	/* suffix may combine with prefix */
    int		ah_follows;	/* another affix block should be following */
    affentry_T	*ah_first;	/* first affix entry */
} affheader_T;


spell_reload_one(
    char_u	*fname,
    int		added_word)	/* invoked through "zg" */
{
    slang_T	*slang;
    int		didit = FALSE;

    for (slang = first_lang; slang != NULL; slang = slang->sl_next)
    {
	if (fullpathcmp(fname, slang->sl_fname, FALSE) == FPC_SAME)
	{
	    slang_clear(slang);
	    if (spell_load_file(fname, NULL, slang, FALSE) == NULL)
		/* reloading failed, clear the language */
		slang_clear(slang);
	    redraw_all_later(SOME_VALID);
	    didit = TRUE;
	}
    }

    /* When "zg" was used and the file wasn't loaded yet, should redo
     * 'spelllang' to load it now. */
    if (added_word && !didit)
	did_set_spelllang(curwin);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


set_spell_chartab(char_u *fol, char_u *low, char_u *upp)
{
    /* We build the new tables here first, so that we can compare with the
     * previous one. */
    spelltab_T	new_st;
    char_u	*pf = fol, *pl = low, *pu = upp;
    int		f, l, u;

    clear_spell_chartab(&new_st);

    while (*pf != NUL)
    {
	if (*pl == NUL || *pu == NUL)
	{
	    EMSG(_(e_affform));
	    return FAIL;
	}
#ifdef FEAT_MBYTE
	f = mb_ptr2char_adv(&pf);
	l = mb_ptr2char_adv(&pl);
	u = mb_ptr2char_adv(&pu);
#else
	f = *pf++;
	l = *pl++;
	u = *pu++;
#endif
	/* Every character that appears is a word character. */
	if (f < 256)
	    new_st.st_isw[f] = TRUE;
	if (l < 256)
	    new_st.st_isw[l] = TRUE;
	if (u < 256)
	    new_st.st_isw[u] = TRUE;

	/* if "LOW" and "FOL" are not the same the "LOW" char needs
	 * case-folding */
	if (l < 256 && l != f)
	{
	    if (f >= 256)
	    {
		EMSG(_(e_affrange));
		return FAIL;
	    }
	    new_st.st_fold[l] = f;
	}

	/* if "UPP" and "FOL" are not the same the "UPP" char needs
	 * case-folding, it's upper case and the "UPP" is the upper case of
	 * "FOL" . */
	if (u < 256 && u != f)
	{
	    if (f >= 256)
	    {
		EMSG(_(e_affrange));
		return FAIL;
	    }
	    new_st.st_fold[u] = f;
	    new_st.st_isu[u] = TRUE;
	    new_st.st_upper[f] = u;
	}
    }

    if (*pl != NUL || *pu != NUL)
    {
	EMSG(_(e_affform));
	return FAIL;
    }

    return set_spell_finish(&new_st);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


spell_load_file(
    char_u	*fname,
    char_u	*lang,
    slang_T	*old_lp,
    int		silent)		/* no error if file doesn't exist */
{
    FILE	*fd;
    char_u	buf[VIMSPELLMAGICL];
    char_u	*p;
    int		i;
    int		n;
    int		len;
    char_u	*save_sourcing_name = sourcing_name;
    linenr_T	save_sourcing_lnum = sourcing_lnum;
    slang_T	*lp = NULL;
    int		c = 0;
    int		res;

    fd = mch_fopen((char *)fname, "r");
    if (fd == NULL)
    {
	if (!silent)
	    EMSG2(_(e_notopen), fname);
	else if (p_verbose > 2)
	{
	    verbose_enter();
	    smsg((char_u *)e_notopen, fname);
	    verbose_leave();
	}
	goto endFAIL;
    }
    if (p_verbose > 2)
    {
	verbose_enter();
	smsg((char_u *)_("Reading spell file \"%s\""), fname);
	verbose_leave();
    }

    if (old_lp == NULL)
    {
	lp = slang_alloc(lang);
	if (lp == NULL)
	    goto endFAIL;

	/* Remember the file name, used to reload the file when it's updated. */
	lp->sl_fname = vim_strsave(fname);
	if (lp->sl_fname == NULL)
	    goto endFAIL;

	/* Check for .add.spl (_add.spl for VMS). */
	lp->sl_add = strstr((char *)gettail(fname), SPL_FNAME_ADD) != NULL;
    }
    else
	lp = old_lp;

    /* Set sourcing_name, so that error messages mention the file name. */
    sourcing_name = fname;
    sourcing_lnum = 0;

    /*
     * <HEADER>: <fileID>
     */
    for (i = 0; i < VIMSPELLMAGICL; ++i)
	buf[i] = getc(fd);				/* <fileID> */
    if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0)
    {
	EMSG(_("E757: This does not look like a spell file"));
	goto endFAIL;
    }
    c = getc(fd);					/* <versionnr> */
    if (c < VIMSPELLVERSION)
    {
	EMSG(_("E771: Old spell file, needs to be updated"));
	goto endFAIL;
    }
    else if (c > VIMSPELLVERSION)
    {
	EMSG(_("E772: Spell file is for newer version of Vim"));
	goto endFAIL;
    }

    /*
     * <SECTIONS>: <section> ... <sectionend>
     * <section>: <sectionID> <sectionflags> <sectionlen> (section contents)
     */
    for (;;)
    {
	n = getc(fd);			    /* <sectionID> or <sectionend> */
	if (n == SN_END)
	    break;
	c = getc(fd);					/* <sectionflags> */
	len = get4c(fd);				/* <sectionlen> */
	if (len < 0)
	    goto truncerr;

	res = 0;
	switch (n)
	{
	    case SN_INFO:
		lp->sl_info = read_string(fd, len);	/* <infotext> */
		if (lp->sl_info == NULL)
		    goto endFAIL;
		break;

	    case SN_REGION:
		res = read_region_section(fd, lp, len);
		break;

	    case SN_CHARFLAGS:
		res = read_charflags_section(fd);
		break;

	    case SN_MIDWORD:
		lp->sl_midword = read_string(fd, len);	/* <midword> */
		if (lp->sl_midword == NULL)
		    goto endFAIL;
		break;

	    case SN_PREFCOND:
		res = read_prefcond_section(fd, lp);
		break;

	    case SN_REP:
		res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first);
		break;

	    case SN_REPSAL:
		res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first);
		break;

	    case SN_SAL:
		res = read_sal_section(fd, lp);
		break;

	    case SN_SOFO:
		res = read_sofo_section(fd, lp);
		break;

	    case SN_MAP:
		p = read_string(fd, len);		/* <mapstr> */
		if (p == NULL)
		    goto endFAIL;
		set_map_str(lp, p);
		vim_free(p);
		break;

	    case SN_WORDS:
		res = read_words_section(fd, lp, len);
		break;

	    case SN_SUGFILE:
		lp->sl_sugtime = get8ctime(fd);		/* <timestamp> */
		break;

	    case SN_NOSPLITSUGS:
		lp->sl_nosplitsugs = TRUE;
		break;

	    case SN_NOCOMPOUNDSUGS:
		lp->sl_nocompoundsugs = TRUE;
		break;

	    case SN_COMPOUND:
		res = read_compound(fd, lp, len);
		break;

	    case SN_NOBREAK:
		lp->sl_nobreak = TRUE;
		break;

	    case SN_SYLLABLE:
		lp->sl_syllable = read_string(fd, len);	/* <syllable> */
		if (lp->sl_syllable == NULL)
		    goto endFAIL;
		if (init_syl_tab(lp) == FAIL)
		    goto endFAIL;
		break;

	    default:
		/* Unsupported section.  When it's required give an error
		 * message.  When it's not required skip the contents. */
		if (c & SNF_REQUIRED)
		{
		    EMSG(_("E770: Unsupported section in spell file"));
		    goto endFAIL;
		}
		while (--len >= 0)
		    if (getc(fd) < 0)
			goto truncerr;
		break;
	}
someerror:
	if (res == SP_FORMERROR)
	{
	    EMSG(_(e_format));
	    goto endFAIL;
	}
	if (res == SP_TRUNCERROR)
	{
truncerr:
	    EMSG(_(e_spell_trunc));
	    goto endFAIL;
	}
	if (res == SP_OTHERERROR)
	    goto endFAIL;
    }

    /* <LWORDTREE> */
    res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fidxs, FALSE, 0);
    if (res != 0)
	goto someerror;

    /* <KWORDTREE> */
    res = spell_read_tree(fd, &lp->sl_kbyts, &lp->sl_kidxs, FALSE, 0);
    if (res != 0)
	goto someerror;

    /* <PREFIXTREE> */
    res = spell_read_tree(fd, &lp->sl_pbyts, &lp->sl_pidxs, TRUE,
							    lp->sl_prefixcnt);
    if (res != 0)
	goto someerror;

    /* For a new file link it in the list of spell files. */
    if (old_lp == NULL && lang != NULL)
    {
	lp->sl_next = first_lang;
	first_lang = lp;
    }

    goto endOK;

endFAIL:
    if (lang != NULL)
	/* truncating the name signals the error to spell_load_lang() */
	*lang = NUL;
    if (lp != NULL && old_lp == NULL)
	slang_free(lp);
    lp = NULL;

endOK:
    if (fd != NULL)
	fclose(fd);
    sourcing_name = save_sourcing_name;
    sourcing_lnum = save_sourcing_lnum;

    return lp;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define VIMSPELLMAGICL 8

#define VIMSPELLVERSION 50

#define SN_REGION	0	/* <regionname> section */

#define SN_CHARFLAGS	1	/* charflags section */

#define SN_MIDWORD	2	/* <midword> section */

#define SN_PREFCOND	3	/* <prefcond> section */

#define SN_REP		4	/* REP items section */

#define SN_SAL		5	/* SAL items section */

#define SN_SOFO		6	/* soundfolding section */

#define SN_MAP		7	/* MAP items section */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SYLLABLE	9	/* syllable section */

#define SN_NOBREAK	10	/* NOBREAK section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_REPSAL	12	/* REPSAL items section */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SN_END		255	/* end of sections */

#define SNF_REQUIRED	1	/* <sectionflags>: required section */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");


sug_write(spellinfo_T *spin, char_u *fname)
{
    FILE	*fd;
    wordnode_T	*tree;
    int		nodecount;
    int		wcount;
    char_u	*line;
    linenr_T	lnum;
    int		len;

    /* Create the file.  Note that an existing file is silently overwritten! */
    fd = mch_fopen((char *)fname, "w");
    if (fd == NULL)
    {
	EMSG2(_(e_notopen), fname);
	return;
    }

    vim_snprintf((char *)IObuff, IOSIZE,
				  _("Writing suggestion file %s ..."), fname);
    spell_message(spin, IObuff);

    /*
     * <SUGHEADER>: <fileID> <versionnr> <timestamp>
     */
    if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) /* <fileID> */
    {
	EMSG(_(e_write));
	goto theend;
    }
    putc(VIMSUGVERSION, fd);				/* <versionnr> */

    /* Write si_sugtime to the file. */
    put_time(fd, spin->si_sugtime);			/* <timestamp> */

    /*
     * <SUGWORDTREE>
     */
    spin->si_memtot = 0;
    tree = spin->si_foldroot->wn_sibling;

    /* Clear the index and wnode fields in the tree. */
    clear_node(tree);

    /* Count the number of nodes.  Needed to be able to allocate the
     * memory when reading the nodes.  Also fills in index for shared
     * nodes. */
    nodecount = put_node(NULL, tree, 0, 0, FALSE);

    /* number of nodes in 4 bytes */
    put_bytes(fd, (long_u)nodecount, 4);	/* <nodecount> */
    spin->si_memtot += nodecount + nodecount * sizeof(int);

    /* Write the nodes. */
    (void)put_node(fd, tree, 0, 0, FALSE);

    /*
     * <SUGTABLE>: <sugwcount> <sugline> ...
     */
    wcount = spin->si_spellbuf->b_ml.ml_line_count;
    put_bytes(fd, (long_u)wcount, 4);	/* <sugwcount> */

    for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum)
    {
	/* <sugline>: <sugnr> ... NUL */
	line = ml_get_buf(spin->si_spellbuf, lnum, FALSE);
	len = (int)STRLEN(line) + 1;
	if (fwrite(line, (size_t)len, (size_t)1, fd) == 0)
	{
	    EMSG(_(e_write));
	    goto theend;
	}
	spin->si_memtot += len;
    }

    /* Write another byte to check for errors. */
    if (putc(0, fd) == EOF)
	EMSG(_(e_write));

    vim_snprintf((char *)IObuff, IOSIZE,
		 _("Estimated runtime memory use: %d bytes"), spin->si_memtot);
    spell_message(spin, IObuff);

theend:
    /* close the file */
    fclose(fd);
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define VIMSPELLMAGIC "VIMspell"  /* string at start of Vim spell file */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static void set_sal_first(slang_T *lp);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


read_rep_section(FILE *fd, garray_T *gap, short *first)
{
    int		cnt;
    fromto_T	*ftp;
    int		i;

    cnt = get2c(fd);					/* <repcount> */
    if (cnt < 0)
	return SP_TRUNCERROR;

    if (ga_grow(gap, cnt) == FAIL)
	return SP_OTHERERROR;

    /* <rep> : <repfromlen> <repfrom> <reptolen> <repto> */
    for (; gap->ga_len < cnt; ++gap->ga_len)
    {
	ftp = &((fromto_T *)gap->ga_data)[gap->ga_len];
	ftp->ft_from = read_cnt_string(fd, 1, &i);
	if (i < 0)
	    return i;
	if (i == 0)
	    return SP_FORMERROR;
	ftp->ft_to = read_cnt_string(fd, 1, &i);
	if (i <= 0)
	{
	    vim_free(ftp->ft_from);
	    if (i < 0)
		return i;
	    return SP_FORMERROR;
	}
    }

    /* Fill the first-index table. */
    for (i = 0; i < 256; ++i)
	first[i] = -1;
    for (i = 0; i < gap->ga_len; ++i)
    {
	ftp = &((fromto_T *)gap->ga_data)[i];
	if (first[*ftp->ft_from] == -1)
	    first[*ftp->ft_from] = i;
    }
    return 0;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


get_affitem(int flagtype, char_u **pp)
{
    int		res;

    if (flagtype == AFT_NUM)
    {
	if (!VIM_ISDIGIT(**pp))
	{
	    ++*pp;	/* always advance, avoid getting stuck */
	    return 0;
	}
	res = getdigits(pp);
    }
    else
    {
#ifdef FEAT_MBYTE
	res = mb_ptr2char_adv(pp);
#else
	res = *(*pp)++;
#endif
	if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG
						 && res >= 'A' && res <= 'Z'))
	{
	    if (**pp == NUL)
		return 0;
#ifdef FEAT_MBYTE
	    res = mb_ptr2char_adv(pp) + (res << 16);
#else
	    res = *(*pp)++ + (res << 16);
#endif
	}
    }
    return res;
}

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;

#define AFT_LONG	1	/* flags are two characters */

#define AFT_CAPLONG	2	/* flags are one or two characters */

#define AFT_NUM		3	/* flags are numbers, comma separated */


flag_in_afflist(int flagtype, char_u *afflist, unsigned flag)
{
    char_u	*p;
    unsigned	n;

    switch (flagtype)
    {
	case AFT_CHAR:
	    return vim_strchr(afflist, flag) != NULL;

	case AFT_CAPLONG:
	case AFT_LONG:
	    for (p = afflist; *p != NUL; )
	    {
#ifdef FEAT_MBYTE
		n = mb_ptr2char_adv(&p);
#else
		n = *p++;
#endif
		if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z'))
								 && *p != NUL)
#ifdef FEAT_MBYTE
		    n = mb_ptr2char_adv(&p) + (n << 16);
#else
		    n = *p++ + (n << 16);
#endif
		if (n == flag)
		    return TRUE;
	    }
	    break;

	case AFT_NUM:
	    for (p = afflist; *p != NUL; )
	    {
		n = getdigits(&p);
		if (n == flag)
		    return TRUE;
		if (*p != NUL)	/* skip over comma */
		    ++p;
	    }
	    break;
    }
    return FALSE;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

typedef struct afffile_S
{
    char_u	*af_enc;	/* "SET", normalized, alloc'ed string or NULL */
    int		af_flagtype;	/* AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG */
    unsigned	af_rare;	/* RARE ID for rare word */
    unsigned	af_keepcase;	/* KEEPCASE ID for keep-case word */
    unsigned	af_bad;		/* BAD ID for banned word */
    unsigned	af_needaffix;	/* NEEDAFFIX ID */
    unsigned	af_circumfix;	/* CIRCUMFIX ID */
    unsigned	af_needcomp;	/* NEEDCOMPOUND ID */
    unsigned	af_comproot;	/* COMPOUNDROOT ID */
    unsigned	af_compforbid;	/* COMPOUNDFORBIDFLAG ID */
    unsigned	af_comppermit;	/* COMPOUNDPERMITFLAG ID */
    unsigned	af_nosuggest;	/* NOSUGGEST ID */
    int		af_pfxpostpone;	/* postpone prefixes without chop string and
				   without flags */
    int		af_ignoreextra;	/* IGNOREEXTRA present */
    hashtab_T	af_pref;	/* hashtable for prefixes, affheader_T */
    hashtab_T	af_suff;	/* hashtable for suffixes, affheader_T */
    hashtab_T	af_comp;	/* hashtable for compound flags, compitem_T */
} afffile_T;


read_charflags_section(FILE *fd)
{
    char_u	*flags;
    char_u	*fol;
    int		flagslen, follen;

    /* <charflagslen> <charflags> */
    flags = read_cnt_string(fd, 1, &flagslen);
    if (flagslen < 0)
	return flagslen;

    /* <folcharslen> <folchars> */
    fol = read_cnt_string(fd, 2, &follen);
    if (follen < 0)
    {
	vim_free(flags);
	return follen;
    }

    /* Set the word-char flags and fill SPELL_ISUPPER() table. */
    if (flags != NULL && fol != NULL)
	set_spell_charflags(flags, flagslen, fol);

    vim_free(flags);
    vim_free(fol);

    /* When <charflagslen> is zero then <fcharlen> must also be zero. */
    if ((flags == NULL) != (fol == NULL))
	return SP_FORMERROR;
    return 0;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define SN_CHARFLAGS	1	/* charflags section */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *e_spell_trunc = N_("E758: Truncated spell file");

static char *e_afftrailing = N_("Trailing text in %s line %d: %s");

static char *e_affname = N_("Affix name too long in %s line %d: %s");

static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP");

static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range");

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


deref_wordnode(spellinfo_T *spin, wordnode_T *node)
{
    wordnode_T	*np;
    int		cnt = 0;

    if (--node->wn_refs == 0)
    {
	for (np = node; np != NULL; np = np->wn_sibling)
	{
	    if (np->wn_child != NULL)
		cnt += deref_wordnode(spin, np->wn_child);
	    free_wordnode(spin, np);
	    ++cnt;
	}
	++cnt;	    /* length field */
    }
    return cnt;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


process_compflags(
    spellinfo_T	*spin,
    afffile_T	*aff,
    char_u	*compflags)
{
    char_u	*p;
    char_u	*prevp;
    unsigned	flag;
    compitem_T	*ci;
    int		id;
    int		len;
    char_u	*tp;
    char_u	key[AH_KEY_LEN];
    hashitem_T	*hi;

    /* Make room for the old and the new compflags, concatenated with a / in
     * between.  Processing it makes it shorter, but we don't know by how
     * much, thus allocate the maximum. */
    len = (int)STRLEN(compflags) + 1;
    if (spin->si_compflags != NULL)
	len += (int)STRLEN(spin->si_compflags) + 1;
    p = getroom(spin, len, FALSE);
    if (p == NULL)
	return;
    if (spin->si_compflags != NULL)
    {
	STRCPY(p, spin->si_compflags);
	STRCAT(p, "/");
    }
    spin->si_compflags = p;
    tp = p + STRLEN(p);

    for (p = compflags; *p != NUL; )
    {
	if (vim_strchr((char_u *)"/?*+[]", *p) != NULL)
	    /* Copy non-flag characters directly. */
	    *tp++ = *p++;
	else
	{
	    /* First get the flag number, also checks validity. */
	    prevp = p;
	    flag = get_affitem(aff->af_flagtype, &p);
	    if (flag != 0)
	    {
		/* Find the flag in the hashtable.  If it was used before, use
		 * the existing ID.  Otherwise add a new entry. */
		vim_strncpy(key, prevp, p - prevp);
		hi = hash_find(&aff->af_comp, key);
		if (!HASHITEM_EMPTY(hi))
		    id = HI2CI(hi)->ci_newID;
		else
		{
		    ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE);
		    if (ci == NULL)
			break;
		    STRCPY(ci->ci_key, key);
		    ci->ci_flag = flag;
		    /* Avoid using a flag ID that has a special meaning in a
		     * regexp (also inside []). */
		    do
		    {
			check_renumber(spin);
			id = spin->si_newcompID--;
		    } while (vim_strchr((char_u *)"/?*+[]\\-^", id) != NULL);
		    ci->ci_newID = id;
		    hash_add(&aff->af_comp, ci->ci_key);
		}
		*tp++ = id;
	    }
	    if (aff->af_flagtype == AFT_NUM && *p == ',')
		++p;
	}
    }

    *tp = NUL;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */


tree_add_word(
    spellinfo_T	*spin,
    char_u	*word,
    wordnode_T	*root,
    int		flags,
    int		region,
    int		affixID)
{
    wordnode_T	*node = root;
    wordnode_T	*np;
    wordnode_T	*copyp, **copyprev;
    wordnode_T	**prev = NULL;
    int		i;

    /* Add each byte of the word to the tree, including the NUL at the end. */
    for (i = 0; ; ++i)
    {
	/* When there is more than one reference to this node we need to make
	 * a copy, so that we can modify it.  Copy the whole list of siblings
	 * (we don't optimize for a partly shared list of siblings). */
	if (node != NULL && node->wn_refs > 1)
	{
	    --node->wn_refs;
	    copyprev = prev;
	    for (copyp = node; copyp != NULL; copyp = copyp->wn_sibling)
	    {
		/* Allocate a new node and copy the info. */
		np = get_wordnode(spin);
		if (np == NULL)
		    return FAIL;
		np->wn_child = copyp->wn_child;
		if (np->wn_child != NULL)
		    ++np->wn_child->wn_refs;	/* child gets extra ref */
		np->wn_byte = copyp->wn_byte;
		if (np->wn_byte == NUL)
		{
		    np->wn_flags = copyp->wn_flags;
		    np->wn_region = copyp->wn_region;
		    np->wn_affixID = copyp->wn_affixID;
		}

		/* Link the new node in the list, there will be one ref. */
		np->wn_refs = 1;
		if (copyprev != NULL)
		    *copyprev = np;
		copyprev = &np->wn_sibling;

		/* Let "node" point to the head of the copied list. */
		if (copyp == node)
		    node = np;
	    }
	}

	/* Look for the sibling that has the same character.  They are sorted
	 * on byte value, thus stop searching when a sibling is found with a
	 * higher byte value.  For zero bytes (end of word) the sorting is
	 * done on flags and then on affixID. */
	while (node != NULL
		&& (node->wn_byte < word[i]
		    || (node->wn_byte == NUL
			&& (flags < 0
			    ? node->wn_affixID < (unsigned)affixID
			    : (node->wn_flags < (unsigned)(flags & WN_MASK)
				|| (node->wn_flags == (flags & WN_MASK)
				    && (spin->si_sugtree
					? (node->wn_region & 0xffff) < region
					: node->wn_affixID
						    < (unsigned)affixID)))))))
	{
	    prev = &node->wn_sibling;
	    node = *prev;
	}
	if (node == NULL
		|| node->wn_byte != word[i]
		|| (word[i] == NUL
		    && (flags < 0
			|| spin->si_sugtree
			|| node->wn_flags != (flags & WN_MASK)
			|| node->wn_affixID != affixID)))
	{
	    /* Allocate a new node. */
	    np = get_wordnode(spin);
	    if (np == NULL)
		return FAIL;
	    np->wn_byte = word[i];

	    /* If "node" is NULL this is a new child or the end of the sibling
	     * list: ref count is one.  Otherwise use ref count of sibling and
	     * make ref count of sibling one (matters when inserting in front
	     * of the list of siblings). */
	    if (node == NULL)
		np->wn_refs = 1;
	    else
	    {
		np->wn_refs = node->wn_refs;
		node->wn_refs = 1;
	    }
	    if (prev != NULL)
		*prev = np;
	    np->wn_sibling = node;
	    node = np;
	}

	if (word[i] == NUL)
	{
	    node->wn_flags = flags;
	    node->wn_region |= region;
	    node->wn_affixID = affixID;
	    break;
	}
	prev = &node->wn_child;
	node = *prev;
    }
#ifdef SPELL_PRINTTREE
    smsg((char_u *)"Added \"%s\"", word);
    spell_print_tree(root->wn_sibling);
#endif

    /* count nr of words added since last message */
    ++spin->si_msg_count;

    if (spin->si_compress_cnt > 1)
    {
	if (--spin->si_compress_cnt == 1)
	    /* Did enough words to lower the block count limit. */
	    spin->si_blocks_cnt += compress_inc;
    }

    /*
     * When we have allocated lots of memory we need to compress the word tree
     * to free up some room.  But compression is slow, and we might actually
     * need that room, thus only compress in the following situations:
     * 1. When not compressed before (si_compress_cnt == 0): when using
     *    "compress_start" blocks.
     * 2. When compressed before and used "compress_inc" blocks before
     *    adding "compress_added" words (si_compress_cnt > 1).
     * 3. When compressed before, added "compress_added" words
     *    (si_compress_cnt == 1) and the number of free nodes drops below the
     *    maximum word length.
     */
#ifndef SPELL_COMPRESS_ALLWAYS
    if (spin->si_compress_cnt == 1
	    ? spin->si_free_count < MAXWLEN
	    : spin->si_blocks_cnt >= compress_start)
#endif
    {
	/* Decrement the block counter.  The effect is that we compress again
	 * when the freed up room has been used and another "compress_inc"
	 * blocks have been allocated.  Unless "compress_added" words have
	 * been added, then the limit is put back again. */
	spin->si_blocks_cnt -= compress_inc;
	spin->si_compress_cnt = compress_added;

	if (spin->si_verbose)
	{
	    msg_start();
	    msg_puts((char_u *)_(msg_compressing));
	    msg_clr_eos();
	    msg_didout = FALSE;
	    msg_col = 0;
	    out_flush();
	}

	/* Compress both trees.  Either they both have many nodes, which makes
	 * compression useful, or one of them is small, which means
	 * compression goes fast.  But when filling the soundfold word tree
	 * there is no keep-case tree. */
	wordtree_compress(spin, spin->si_foldroot);
	if (affixID >= 0)
	    wordtree_compress(spin, spin->si_keeproot);
    }

    return OK;
}

#define BY_NOFLAGS	0	/* end of word without flags or region; for
				 * postponed prefix: no <pflags> */

#define BY_INDEX	1	/* child is shared, index follows */

#define BY_FLAGS	2	/* end of word, <flags> byte follows; for
				 * postponed prefix: <pflags> follows */

#define BY_FLAGS2	3	/* end of word, <flags> and <flags2> bytes
				 * follow; never used in prefix tree */

#define BY_SPECIAL  BY_FLAGS2	/* highest special byte value */

#define SN_COMPOUND	8	/* compound words section */

#define SN_SUGFILE	11	/* timestamp for .sug file */

#define SN_WORDS	13	/* common words */

#define SN_NOSPLITSUGS	14	/* don't split word for suggestions */

#define SN_INFO		15	/* info section */

#define SN_NOCOMPOUNDSUGS 16	/* don't compound for suggestions */

#define SN_END		255	/* end of sections */

static int set_spell_finish(spelltab_T	*new_st);

static int write_spell_prefcond(FILE *fd, garray_T *gap);

static char_u *read_cnt_string(FILE *fd, int cnt_bytes, int *lenp);

static int read_region_section(FILE *fd, slang_T *slang, int len);

static int read_charflags_section(FILE *fd);

static int read_prefcond_section(FILE *fd, slang_T *lp);

static int read_rep_section(FILE *fd, garray_T *gap, short *first);

static int read_sal_section(FILE *fd, slang_T *slang);

static int read_words_section(FILE *fd, slang_T *lp, int len);

static int read_sofo_section(FILE *fd, slang_T *slang);

static int read_compound(FILE *fd, slang_T *slang, int len);

static int set_sofo(slang_T *lp, char_u *from, char_u *to);

static int *mb_str2wide(char_u *s);

static int spell_read_tree(FILE *fd, char_u **bytsp, idx_T **idxsp, int prefixtree, int prefixcnt);

static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr);

static void spell_reload_one(char_u *fname, int added_word);

static void set_spell_charflags(char_u *flags, int cnt, char_u *upp);

static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp);

static void set_map_str(slang_T *lp, char_u *map);

static char *msg_compressing = N_("Compressing word tree...");

#define MAXLINELEN  500		/* Maximum length in bytes of a line in a .aff
				   and .dic file. */