canonical_charset_name(const char *charset)
{
	char cs[16];
	char *p;
	const char *s;

	if (charset == NULL || charset[0] == '\0'
	    || strlen(charset) > 15)
		return (charset);

	/* Copy name to uppercase. */
	p = cs;
	s = charset;
	while (*s) {
		char c = *s++;
		if (c >= 'a' && c <= 'z')
			c -= 'a' - 'A';
		*p++ = c;
	}
	*p++ = '\0';

	if (strcmp(cs, "UTF-8") == 0 ||
	    strcmp(cs, "UTF8") == 0)
		return ("UTF-8");
	if (strcmp(cs, "UTF-16BE") == 0 ||
	    strcmp(cs, "UTF16BE") == 0)
		return ("UTF-16BE");
	if (strcmp(cs, "UTF-16LE") == 0 ||
	    strcmp(cs, "UTF16LE") == 0)
		return ("UTF-16LE");
	if (strcmp(cs, "CP932") == 0)
		return ("CP932");
	return (charset);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);

static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
    const wchar_t *, size_t, struct archive_string_conv *);


get_sconv_object(struct archive *a, const char *fc, const char *tc, int flag)
{
	struct archive_string_conv *sc;
	unsigned current_codepage;

	/* Check if we have made the sconv object. */
	sc = find_sconv_object(a, fc, tc);
	if (sc != NULL)
		return (sc);

	if (a == NULL)
		current_codepage = get_current_codepage();
	else
		current_codepage = a->current_codepage;

	sc = create_sconv_object(canonical_charset_name(fc),
	    canonical_charset_name(tc), current_codepage, flag);
	if (sc == NULL) {
		if (a != NULL)
			archive_set_error(a, ENOMEM,
			    "Could not allocate memory for "
			    "a string conversion object");
		return (NULL);
	}

	/*
	 * If there is no converter for current string conversion object,
	 * we cannot handle this conversion.
	 */
	if (sc->nconverter == 0) {
		if (a != NULL) {
#if HAVE_ICONV
			archive_set_error(a, ARCHIVE_ERRNO_MISC,
			    "iconv_open failed : Cannot handle ``%s''",
			    (flag & SCONV_TO_CHARSET)?tc:fc);
#else
			archive_set_error(a, ARCHIVE_ERRNO_MISC,
			    "A character-set conversion not fully supported "
			    "on this platform");
#endif
		}
		/* Failed; free a sconv object. */
		free_sconv_object(sc);
		return (NULL);
	}

	/*
	 * Success!
	 */
	if (a != NULL)
		add_sconv_object(a, sc);
	return (sc);
}

Fix a possible heap-buffer-overflow in archive_string_append_from_wcs()

When we grow the archive_string buffer, we have to make sure it fits
at least one maximum-sized multibyte character in the current locale
and the null character.

Fixes #1298

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static unsigned get_current_codepage(void);

static unsigned get_current_oemcp(void);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


archive_string_append(struct archive_string *as, const char *p, size_t s)
{
	if (archive_string_ensure(as, as->length + s + 1) == NULL)
		return (NULL);
	if (s)
		memmove(as->s + as->length, p, s);
	as->length += s;
	as->s[as->length] = 0;
	return (as);
}

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


utf16le_to_unicode(uint32_t *pwc, const char *s, size_t n)
{
	return (utf16_to_unicode(pwc, s, n, 0));
}


archive_mstring_copy(struct archive_mstring *dest, struct archive_mstring *src)
{
	dest->aes_set = src->aes_set;
	archive_string_copy(&(dest->aes_mbs), &(src->aes_mbs));
	archive_string_copy(&(dest->aes_utf8), &(src->aes_utf8));
	archive_wstring_copy(&(dest->aes_wcs), &(src->aes_wcs));
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);

static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
    const wchar_t *, size_t, struct archive_string_conv *);

static int strncat_in_codepage(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);


archive_string_concat(struct archive_string *dest, struct archive_string *src)
{
	if (archive_string_append(dest, src->s, src->length) == NULL)
		__archive_errx(1, "Out of memory");
}


archive_mstring_get_wcs(struct archive *a, struct archive_mstring *aes,
    const wchar_t **wp)
{
	int r, ret = 0;

	(void)a;/* UNUSED */
	/* Return WCS form if we already have it. */
	if (aes->aes_set & AES_SET_WCS) {
		*wp = aes->aes_wcs.s;
		return (ret);
	}

	*wp = NULL;
	/* Try converting MBS to WCS using native locale. */
	if (aes->aes_set & AES_SET_MBS) {
		archive_wstring_empty(&(aes->aes_wcs));
		r = archive_wstring_append_from_mbs(&(aes->aes_wcs),
		    aes->aes_mbs.s, aes->aes_mbs.length);
		if (r == 0) {
			aes->aes_set |= AES_SET_WCS;
			*wp = aes->aes_wcs.s;
		} else
			ret = -1;/* failure. */
	}
	return (ret);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);


combine_surrogate_pair(uint32_t uc, uint32_t uc2)
{
	uc -= 0xD800;
	uc *= 0x400;
	uc += uc2 - 0xDC00;
	uc += 0x10000;
	return (uc);
}

static int _utf8_to_unicode(uint32_t *, const char *, size_t);

static int utf8_to_unicode(uint32_t *, const char *, size_t);

static inline uint32_t combine_surrogate_pair(uint32_t, uint32_t);

static int cesu8_to_unicode(uint32_t *, const char *, size_t);

static size_t unicode_to_utf8(char *, size_t, uint32_t);

static int utf16_to_unicode(uint32_t *, const char *, size_t, int);

static size_t unicode_to_utf16be(char *, size_t, uint32_t);

static size_t unicode_to_utf16le(char *, size_t, uint32_t);

#define REPLACE_UC_WITH_UC2() do {		\
	uc = uc2;				\
	ucptr = uc2ptr;				\
	n = n2;					\
} while (0)

#define EXPAND_BUFFER() do {			\
	as->length = p - as->s;			\
	if (archive_string_ensure(as,		\
	    as->buffer_length + len * tm + ts) == NULL)\
		return (-1);			\
	p = as->s + as->length;			\
	endp = as->s + as->buffer_length - ts;	\
} while (0)


win_strncat_to_utf16(struct archive_string *as16, const void *_p,
    size_t length, struct archive_string_conv *sc, int bigendian)
{
	const char *s = (const char *)_p;
	char *u16;
	size_t count, avail;

	if (archive_string_ensure(as16,
	    as16->length + (length + 1) * 2) == NULL)
		return (-1);

	u16 = as16->s + as16->length;
	avail = as16->buffer_length - 2;
	if (sc->from_cp == CP_C_LOCALE) {
		/*
		 * "C" locale special process.
		 */
		count = 0;
		while (count < length && *s) {
			if (bigendian)
				archive_be16enc(u16, *s);
			else
				archive_le16enc(u16, *s);
			u16 += 2;
			s++;
			count++;
		}
		as16->length += count << 1;
		as16->s[as16->length] = 0;
		as16->s[as16->length+1] = 0;
		return (0);
	}
	do {
		count = MultiByteToWideChar(sc->from_cp,
		    MB_PRECOMPOSED, s, (int)length, (LPWSTR)u16, (int)avail>>1);
		/* Exit loop if we succeeded */
		if (count != 0 ||
		    GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
			break;
		}
		/* Expand buffer and try again */
		count = MultiByteToWideChar(sc->from_cp,
		    MB_PRECOMPOSED, s, (int)length, NULL, 0);
		if (archive_string_ensure(as16, (count +1) * 2)
		    == NULL)
			return (-1);
		u16 = as16->s + as16->length;
		avail = as16->buffer_length - 2;
	} while (1);
	as16->length += count * 2;
	as16->s[as16->length] = 0;
	as16->s[as16->length+1] = 0;
	if (count == 0)
		return (-1);

	if (is_big_endian()) {
		if (!bigendian) {
			while (count > 0) {
				uint16_t v = archive_be16dec(u16);
				archive_le16enc(u16, v);
				u16 += 2;
				count--;
			}
		}
	} else {
		if (bigendian) {
			while (count > 0) {
				uint16_t v = archive_le16dec(u16);
				archive_be16enc(u16, v);
				u16 += 2;
				count--;
			}
		}
	}
	return (0);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static unsigned get_current_codepage(void);

static unsigned get_current_oemcp(void);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


add_sconv_object(struct archive *a, struct archive_string_conv *sc)
{
	struct archive_string_conv **psc;

	/* Add a new sconv to sconv list. */
	psc = &(a->sconv);
	while (*psc != NULL)
		psc = &((*psc)->next);
	*psc = sc;
}


utf8_to_unicode(uint32_t *pwc, const char *s, size_t n)
{
	int cnt;

	cnt = _utf8_to_unicode(pwc, s, n);
	/* Any of Surrogate pair is not legal Unicode values. */
	if (cnt == 3 && IS_SURROGATE_PAIR_LA(*pwc))
		return (-3);
	return (cnt);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF)

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);


utf16be_to_unicode(uint32_t *pwc, const char *s, size_t n)
{
	return (utf16_to_unicode(pwc, s, n, 1));
}


archive_string_conversion_from_charset(struct archive *a, const char *charset,
    int best_effort)
{
	int flag = SCONV_FROM_CHARSET;

	if (best_effort)
		flag |= SCONV_BEST_EFFORT;
	return (get_sconv_object(a, charset, get_current_charset(a), flag));
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);


get_current_oemcp(void)
{
	return (-1);/* Unknown */
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static void free_sconv_object(struct archive_string_conv *);

static unsigned get_current_codepage(void);

static unsigned get_current_oemcp(void);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int is_big_endian(void);

static int strncat_in_codepage(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);

static int win_strncat_from_utf16be(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);

static int win_strncat_from_utf16le(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);

static int win_strncat_to_utf16be(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);


archive_mstring_get_mbs(struct archive *a, struct archive_mstring *aes,
    const char **p)
{
	int r, ret = 0;

	(void)a; /* UNUSED */
	/* If we already have an MBS form, return that immediately. */
	if (aes->aes_set & AES_SET_MBS) {
		*p = aes->aes_mbs.s;
		return (ret);
	}

	*p = NULL;
	/* If there's a WCS form, try converting with the native locale. */
	if (aes->aes_set & AES_SET_WCS) {
		archive_string_empty(&(aes->aes_mbs));
		r = archive_string_append_from_wcs(&(aes->aes_mbs),
		    aes->aes_wcs.s, aes->aes_wcs.length);
		*p = aes->aes_mbs.s;
		if (r == 0) {
			aes->aes_set |= AES_SET_MBS;
			return (ret);
		} else
			ret = -1;
	}

	/*
	 * Only a UTF-8 form cannot avail because its conversion already
	 * failed at archive_mstring_update_utf8().
	 */
	return (ret);
}


archive_string_default_conversion_for_read(struct archive *a)
{
	const char *cur_charset = get_current_charset(a);
	char oemcp[16];

	/* NOTE: a check of cur_charset is unneeded but we need
	 * that get_current_charset() has been surely called at
	 * this time whatever C compiler optimized. */
	if (cur_charset != NULL &&
	    (a->current_codepage == CP_C_LOCALE ||
	     a->current_codepage == a->current_oemcp))
		return (NULL);/* no conversion. */

	_snprintf(oemcp, sizeof(oemcp)-1, "CP%d", a->current_oemcp);
	/* Make sure a null termination must be set. */
	oemcp[sizeof(oemcp)-1] = '\0';
	return (get_sconv_object(a, oemcp, cur_charset,
	    SCONV_FROM_CHARSET));
}


free_sconv_object(struct archive_string_conv *sc)
{
	free(sc->from_charset);
	free(sc->to_charset);
	archive_string_free(&sc->utftmp);
#if HAVE_ICONV
	if (sc->cd != (iconv_t)-1)
		iconv_close(sc->cd);
	if (sc->cd_w != (iconv_t)-1)
		iconv_close(sc->cd_w);
#endif
	free(sc);
}


invalid_mbs(const void *_p, size_t n, struct archive_string_conv *sc)
{
	const char *p = (const char *)_p;
	unsigned codepage;
	DWORD mbflag = MB_ERR_INVALID_CHARS;

	if (sc->flag & SCONV_FROM_CHARSET)
		codepage = sc->to_cp;
	else
		codepage = sc->from_cp;

	if (codepage == CP_C_LOCALE)
		return (0);
	if (codepage != CP_UTF8)
		mbflag |= MB_PRECOMPOSED;

	if (MultiByteToWideChar(codepage, mbflag, p, (int)n, NULL, 0) == 0)
		return (-1); /* Invalid */
	return (0); /* Okay */
}


get_current_codepage(void)
{
	return (-1);/* Unknown */
}


archive_mstring_copy_mbs(struct archive_mstring *aes, const char *mbs)
{
	if (mbs == NULL) {
		aes->aes_set = 0;
		return (0);
	}
	return (archive_mstring_copy_mbs_len(aes, mbs, strlen(mbs)));
}


archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
    const char *s, size_t length, struct archive_string_conv *sc)
{
	int count, ret = 0;
	UINT from_cp;

	if (sc != NULL)
		from_cp = sc->from_cp;
	else
		from_cp = get_current_codepage();

	if (from_cp == CP_C_LOCALE) {
		/*
		 * "C" locale special processing.
		 */
		wchar_t *ws;
		const unsigned char *mp;

		if (NULL == archive_wstring_ensure(dest,
		    dest->length + length + 1))
			return (-1);

		ws = dest->s + dest->length;
		mp = (const unsigned char *)s;
		count = 0;
		while (count < (int)length && *mp) {
			*ws++ = (wchar_t)*mp++;
			count++;
		}
	} else if (sc != NULL &&
	    (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) {
		/*
		 * Normalize UTF-8 and UTF-16BE and convert it directly
		 * to UTF-16 as wchar_t.
		 */
		struct archive_string u16;
		int saved_flag = sc->flag;/* save current flag. */

		if (is_big_endian())
			sc->flag |= SCONV_TO_UTF16BE;
		else
			sc->flag |= SCONV_TO_UTF16LE;

		if (sc->flag & SCONV_FROM_UTF16) {
			/*
			 *  UTF-16BE/LE NFD ===> UTF-16 NFC
			 *  UTF-16BE/LE NFC ===> UTF-16 NFD
			 */
			count = (int)utf16nbytes(s, length);
		} else {
			/*
			 *  UTF-8 NFD ===> UTF-16 NFC
			 *  UTF-8 NFC ===> UTF-16 NFD
			 */
			count = (int)mbsnbytes(s, length);
		}
		u16.s = (char *)dest->s;
		u16.length = dest->length << 1;;
		u16.buffer_length = dest->buffer_length;
		if (sc->flag & SCONV_NORMALIZATION_C)
			ret = archive_string_normalize_C(&u16, s, count, sc);
		else
			ret = archive_string_normalize_D(&u16, s, count, sc);
		dest->s = (wchar_t *)u16.s;
		dest->length = u16.length >> 1;
		dest->buffer_length = u16.buffer_length;
		sc->flag = saved_flag;/* restore the saved flag. */
		return (ret);
	} else if (sc != NULL && (sc->flag & SCONV_FROM_UTF16)) {
		count = (int)utf16nbytes(s, length);
		count >>= 1; /* to be WCS length */
		/* Allocate memory for WCS. */
		if (NULL == archive_wstring_ensure(dest,
		    dest->length + count + 1))
			return (-1);
		wmemcpy(dest->s + dest->length, (const wchar_t *)s, count);
		if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) {
			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
			int b;
			for (b = 0; b < count; b++) {
				uint16_t val = archive_le16dec(u16+b);
				archive_be16enc(u16+b, val);
			}
		} else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) {
			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
			int b;
			for (b = 0; b < count; b++) {
				uint16_t val = archive_be16dec(u16+b);
				archive_le16enc(u16+b, val);
			}
		}
	} else {
		DWORD mbflag;
		size_t buffsize;

		if (sc == NULL)
			mbflag = 0;
		else if (sc->flag & SCONV_FROM_CHARSET) {
			/* Do not trust the length which comes from
			 * an archive file. */
			length = mbsnbytes(s, length);
			mbflag = 0;
		} else
			mbflag = MB_PRECOMPOSED;

		buffsize = dest->length + length + 1;
		do {
			/* Allocate memory for WCS. */
			if (NULL == archive_wstring_ensure(dest, buffsize))
				return (-1);
			/* Convert MBS to WCS. */
			count = MultiByteToWideChar(from_cp,
			    mbflag, s, (int)length, dest->s + dest->length,
			    (int)(dest->buffer_length >> 1) -1);
			if (count == 0 &&
			    GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
				/* Expand the WCS buffer. */
				buffsize = dest->buffer_length << 1;
				continue;
			}
			if (count == 0 && length != 0)
				ret = -1;
			break;
		} while (1);
	}
	dest->length += count;
	dest->s[dest->length] = L'\0';
	return (ret);
}


make_codepage_from_charset(const char *charset)
{
	char cs[16];
	char *p;
	unsigned cp;
	int a, b;

	if (charset == NULL || strlen(charset) > 15)
		return -1;

	/* Copy name to uppercase. */
	p = cs;
	while (*charset) {
		char c = *charset++;
		if (c >= 'a' && c <= 'z')
			c -= 'a' - 'A';
		*p++ = c;
	}
	*p++ = '\0';
	cp = -1;

	/* Look it up in the table first, so that we can easily
	 * override CP367, which we map to 1252 instead of 367. */
	a = 0;
	b = sizeof(charsets)/sizeof(charsets[0]);
	while (b > a) {
		int c = (b + a) / 2;
		int r = strcmp(charsets[c].name, cs);
		if (r < 0)
			a = c + 1;
		else if (r > 0)
			b = c;
		else
			return charsets[c].cp;
	}

	/* If it's not in the table, try to parse it. */
	switch (*cs) {
	case 'C':
		if (cs[1] == 'P' && cs[2] >= '0' && cs[2] <= '9') {
			cp = my_atoi(cs + 2);
		} else if (strcmp(cs, "CP_ACP") == 0)
			cp = get_current_codepage();
		else if (strcmp(cs, "CP_OEMCP") == 0)
			cp = get_current_oemcp();
		break;
	case 'I':
		if (cs[1] == 'B' && cs[2] == 'M' &&
		    cs[3] >= '0' && cs[3] <= '9') {
			cp = my_atoi(cs + 3);
		}
		break;
	case 'W':
		if (strncmp(cs, "WINDOWS-", 8) == 0) {
			cp = my_atoi(cs + 8);
			if (cp != 874 && (cp < 1250 || cp > 1258))
				cp = -1;/* This may invalid code. */
		}
		break;
	}
	return (cp);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static unsigned get_current_codepage(void);

static unsigned get_current_oemcp(void);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


get_current_codepage(void)
{
	char *locale, *p;
	unsigned cp;

	locale = setlocale(LC_CTYPE, NULL);
	if (locale == NULL)
		return (GetACP());
	if (locale[0] == 'C' && locale[1] == '\0')
		return (CP_C_LOCALE);
	p = strrchr(locale, '.');
	if (p == NULL)
		return (GetACP());
	if (strcmp(p+1, "utf8") == 0)
		return CP_UTF8;
	cp = my_atoi(p+1);
	if ((int)cp <= 0)
		return (GetACP());
	return (cp);
}


archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs)
{
	return archive_mstring_copy_wcs_len(aes, wcs,
				wcs == NULL ? 0 : wcslen(wcs));
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);


strncat_from_utf8_to_utf8(struct archive_string *as, const void *_p,
    size_t len, struct archive_string_conv *sc)
{
	const char *s;
	char *p, *endp;
	int n, ret = 0;

	(void)sc; /* UNUSED */

	if (archive_string_ensure(as, as->length + len + 1) == NULL)
		return (-1);

	s = (const char *)_p;
	p = as->s + as->length;
	endp = as->s + as->buffer_length -1;
	do {
		uint32_t uc;
		const char *ss = s;
		size_t w;

		/*
		 * Forward byte sequence until a conversion of that is needed.
		 */
		while ((n = utf8_to_unicode(&uc, s, len)) > 0) {
			s += n;
			len -= n;
		}
		if (ss < s) {
			if (p + (s - ss) > endp) {
				as->length = p - as->s;
				if (archive_string_ensure(as,
				    as->buffer_length + len + 1) == NULL)
					return (-1);
				p = as->s + as->length;
				endp = as->s + as->buffer_length -1;
			}

			memcpy(p, ss, s - ss);
			p += s - ss;
		}

		/*
		 * If n is negative, current byte sequence needs a replacement.
		 */
		if (n < 0) {
			if (n == -3 && IS_SURROGATE_PAIR_LA(uc)) {
				/* Current byte sequence may be CESU-8. */
				n = cesu8_to_unicode(&uc, s, len);
			}
			if (n < 0) {
				ret = -1;
				n *= -1;/* Use a replaced unicode character. */
			}

			/* Rebuild UTF-8 byte sequence. */
			while ((w = unicode_to_utf8(p, endp - p, uc)) == 0) {
				as->length = p - as->s;
				if (archive_string_ensure(as,
				    as->buffer_length + len + 1) == NULL)
					return (-1);
				p = as->s + as->length;
				endp = as->s + as->buffer_length -1;
			}
			p += w;
			s += n;
			len -= n;
		}
	} while (n > 0);
	as->length = p - as->s;
	as->s[as->length] = '\0';
	return (ret);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define IS_SURROGATE_PAIR_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDFFF)

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);


archive_strncat(struct archive_string *as, const void *_p, size_t n)
{
	size_t s;
	const char *p, *pp;

	p = (const char *)_p;

	/* Like strlen(p), except won't examine positions beyond p[n]. */
	s = 0;
	pp = p;
	while (s < n && *pp) {
		pp++;
		s++;
	}
	if ((as = archive_string_append(as, p, s)) == NULL)
		__archive_errx(1, "Out of memory");
	return (as);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static unsigned get_current_codepage(void);

static unsigned get_current_oemcp(void);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);


archive_wstrappend_wchar(struct archive_wstring *as, wchar_t c)
{
	if ((as = archive_wstring_append(as, &c, 1)) == NULL)
		__archive_errx(1, "Out of memory");
	return (as);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);


default_iconv_charset(const char *charset) {
	if (charset != NULL && charset[0] != '\0')
		return charset;
#if HAVE_LOCALE_CHARSET && !defined(__APPLE__)
	/* locale_charset() is broken on Mac OS */
	return locale_charset();
#elif HAVE_NL_LANGINFO
	return nl_langinfo(CODESET);
#else
	return "";
#endif
}


archive_array_append(struct archive_string *as, const char *p, size_t s)
{
	return archive_string_append(as, p, s);
}


archive_strappend_char(struct archive_string *as, char c)
{
	if ((as = archive_string_append(as, &c, 1)) == NULL)
		__archive_errx(1, "Out of memory");
	return (as);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);

static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
    const wchar_t *, size_t, struct archive_string_conv *);


utf16nbytes(const void *_p, size_t n)
{
	size_t s;
	const char *p, *pp;

	if (_p == NULL)
		return (0);
	p = (const char *)_p;

	/* Like strlen(p), except won't examine positions beyond p[n]. */
	s = 0;
	pp = p;
	n >>= 1;
	while (s < n && (pp[0] || pp[1])) {
		pp += 2;
		s++;
	}
	return (s<<1);
}


best_effort_strncat_from_utf16le(struct archive_string *as, const void *_p,
    size_t bytes, struct archive_string_conv *sc)
{
	return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 0));
}


archive_mstring_get_mbs_l(struct archive_mstring *aes,
    const char **p, size_t *length, struct archive_string_conv *sc)
{
	int r, ret = 0;

#if defined(_WIN32) && !defined(__CYGWIN__)
	/*
	 * Internationalization programming on Windows must use Wide
	 * characters because Windows platform cannot make locale UTF-8.
	 */
	if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) {
		archive_string_empty(&(aes->aes_mbs_in_locale));
		r = archive_string_append_from_wcs_in_codepage(
		    &(aes->aes_mbs_in_locale), aes->aes_wcs.s,
		    aes->aes_wcs.length, sc);
		if (r == 0) {
			*p = aes->aes_mbs_in_locale.s;
			if (length != NULL)
				*length = aes->aes_mbs_in_locale.length;
			return (0);
		} else if (errno == ENOMEM)
			return (-1);
		else
			ret = -1;
	}
#endif

	/* If there is not an MBS form but is a WCS form, try converting
	 * with the native locale to be used for translating it to specified
	 * character-set. */
	if ((aes->aes_set & AES_SET_MBS) == 0 &&
	    (aes->aes_set & AES_SET_WCS) != 0) {
		archive_string_empty(&(aes->aes_mbs));
		r = archive_string_append_from_wcs(&(aes->aes_mbs),
		    aes->aes_wcs.s, aes->aes_wcs.length);
		if (r == 0)
			aes->aes_set |= AES_SET_MBS;
		else if (errno == ENOMEM)
			return (-1);
		else
			ret = -1;
	}
	/* If we already have an MBS form, use it to be translated to
	 * specified character-set. */
	if (aes->aes_set & AES_SET_MBS) {
		if (sc == NULL) {
			/* Conversion is unneeded. */
			*p = aes->aes_mbs.s;
			if (length != NULL)
				*length = aes->aes_mbs.length;
			return (0);
		}
		ret = archive_strncpy_l(&(aes->aes_mbs_in_locale),
		    aes->aes_mbs.s, aes->aes_mbs.length, sc);
		*p = aes->aes_mbs_in_locale.s;
		if (length != NULL)
			*length = aes->aes_mbs_in_locale.length;
	} else {
		*p = NULL;
		if (length != NULL)
			*length = 0;
	}
	return (ret);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


cesu8_to_unicode(uint32_t *pwc, const char *s, size_t n)
{
	uint32_t wc = 0;
	int cnt;

	cnt = _utf8_to_unicode(&wc, s, n);
	if (cnt == 3 && IS_HIGH_SURROGATE_LA(wc)) {
		uint32_t wc2 = 0;
		if (n - 3 < 3) {
			/* Invalid byte sequence. */
			goto invalid_sequence;
		}
		cnt = _utf8_to_unicode(&wc2, s+3, n-3);
		if (cnt != 3 || !IS_LOW_SURROGATE_LA(wc2)) {
			/* Invalid byte sequence. */
			goto invalid_sequence;
		}
		wc = combine_surrogate_pair(wc, wc2);
		cnt = 6;
	} else if (cnt == 3 && IS_LOW_SURROGATE_LA(wc)) {
		/* Invalid byte sequence. */
		goto invalid_sequence;
	}
	*pwc = wc;
	return (cnt);
invalid_sequence:
	*pwc = UNICODE_R_CHAR;/* set the Replacement Character instead. */
	if (cnt > 0)
		cnt *= -1;
	return (cnt);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define IS_HIGH_SURROGATE_LA(uc) ((uc) >= 0xD800 && (uc) <= 0xDBFF)

#define IS_LOW_SURROGATE_LA(uc)	 ((uc) >= 0xDC00 && (uc) <= 0xDFFF)

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);


get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc)
{
	int t, b;

	/*
	 * These are not converted to NFD on Mac OS.
	 */
	if ((uc >= 0x2000 && uc <= 0x2FFF) ||
	    (uc >= 0xF900 && uc <= 0xFAFF) ||
	    (uc >= 0x2F800 && uc <= 0x2FAFF))
		return (0);
	/*
	 * Those code points are not converted to NFD on Mac OS.
	 * I do not know the reason because it is undocumented.
	 *   NFC        NFD
	 *   1109A  ==> 11099 110BA
	 *   1109C  ==> 1109B 110BA
	 *   110AB  ==> 110A5 110BA
	 */
	if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB)
		return (0);

	t = 0;
	b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1;
	while (b >= t) {
		int m = (t + b) / 2;
		if (u_decomposition_table[m].nfc < uc)
			t = m + 1;
		else if (u_decomposition_table[m].nfc > uc)
			b = m - 1;
		else {
			*cp1 = u_decomposition_table[m].cp1;
			*cp2 = u_decomposition_table[m].cp2;
			return (1);
		}
	}
	return (0);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);

static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
    const wchar_t *, size_t, struct archive_string_conv *);

static int is_big_endian(void);

static int strncat_in_codepage(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);


add_converter(struct archive_string_conv *sc, int (*converter)
    (struct archive_string *, const void *, size_t,
     struct archive_string_conv *))
{
	if (sc == NULL || sc->nconverter >= 2)
		__archive_errx(1, "Programming error");
	sc->converter[sc->nconverter++] = converter;
}


win_strncat_to_utf16le(struct archive_string *as16, const void *_p,
    size_t length, struct archive_string_conv *sc)
{
	return (win_strncat_to_utf16(as16, _p, length, sc, 0));
}


win_strncat_from_utf16le(struct archive_string *as, const void *_p,
    size_t bytes, struct archive_string_conv *sc)
{
	return (win_strncat_from_utf16(as, _p, bytes, sc, 0));
}


archive_mstring_copy_mbs_len(struct archive_mstring *aes, const char *mbs,
    size_t len)
{
	if (mbs == NULL) {
		aes->aes_set = 0;
		return (0);
	}
	aes->aes_set = AES_SET_MBS; /* Only MBS form is set now. */
	archive_strncpy(&(aes->aes_mbs), mbs, len);
	archive_string_empty(&(aes->aes_utf8));
	archive_wstring_empty(&(aes->aes_wcs));
	return (0);
}


archive_mstring_clean(struct archive_mstring *aes)
{
	archive_wstring_free(&(aes->aes_wcs));
	archive_string_free(&(aes->aes_mbs));
	archive_string_free(&(aes->aes_utf8));
	archive_string_free(&(aes->aes_mbs_in_locale));
	aes->aes_set = 0;
}


best_effort_strncat_to_utf16le(struct archive_string *as16, const void *_p,
    size_t length, struct archive_string_conv *sc)
{
	return (best_effort_strncat_to_utf16(as16, _p, length, sc, 0));
}


strncat_from_utf8_libarchive2(struct archive_string *as,
    const void *_p, size_t len, struct archive_string_conv *sc)
{
	const char *s;
	int n;
	char *p;
	char *end;
	uint32_t unicode;
#if HAVE_WCRTOMB
	mbstate_t shift_state;

	memset(&shift_state, 0, sizeof(shift_state));
#else
	/* Clear the shift state before starting. */
	wctomb(NULL, L'\0');
#endif
	(void)sc; /* UNUSED */
	/*
	 * Allocate buffer for MBS.
	 * We need this allocation here since it is possible that
	 * as->s is still NULL.
	 */
	if (archive_string_ensure(as, as->length + len + 1) == NULL)
		return (-1);

	s = (const char *)_p;
	p = as->s + as->length;
	end = as->s + as->buffer_length - MB_CUR_MAX -1;
	while ((n = _utf8_to_unicode(&unicode, s, len)) != 0) {
		wchar_t wc;

		if (p >= end) {
			as->length = p - as->s;
			/* Re-allocate buffer for MBS. */
			if (archive_string_ensure(as,
			    as->length + max(len * 2,
			    (size_t)MB_CUR_MAX) + 1) == NULL)
				return (-1);
			p = as->s + as->length;
			end = as->s + as->buffer_length - MB_CUR_MAX -1;
		}

		/*
		 * As libarchive 2.x, translates the UTF-8 characters into
		 * wide-characters in the assumption that WCS is Unicode.
		 */
		if (n < 0) {
			n *= -1;
			wc = L'?';
		} else
			wc = (wchar_t)unicode;

		s += n;
		len -= n;
		/*
		 * Translates the wide-character into the current locale MBS.
		 */
#if HAVE_WCRTOMB
		n = (int)wcrtomb(p, wc, &shift_state);
#else
		n = (int)wctomb(p, wc);
#endif
		if (n == -1)
			return (-1);
		p += n;
	}
	as->length = p - as->s;
	as->s[as->length] = '\0';
	return (0);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

#define max(a, b)       ((a)>(b)?(a):(b))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);


get_nfc(uint32_t uc, uint32_t uc2)
{
	int t, b;

	t = 0;
	b = sizeof(u_composition_table)/sizeof(u_composition_table[0]) -1;
	while (b >= t) {
		int m = (t + b) / 2;
		if (u_composition_table[m].cp1 < uc)
			t = m + 1;
		else if (u_composition_table[m].cp1 > uc)
			b = m - 1;
		else if (u_composition_table[m].cp2 < uc2)
			t = m + 1;
		else if (u_composition_table[m].cp2 > uc2)
			b = m - 1;
		else
			return (u_composition_table[m].nfc);
	}
	return (0);
}


find_sconv_object(struct archive *a, const char *fc, const char *tc)
{
	struct archive_string_conv *sc;

	if (a == NULL)
		return (NULL);

	for (sc = a->sconv; sc != NULL; sc = sc->next) {
		if (strcmp(sc->from_charset, fc) == 0 &&
		    strcmp(sc->to_charset, tc) == 0)
			break;
	}
	return (sc);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static unsigned make_codepage_from_charset(const char *);

static size_t mbsnbytes(const void *, size_t);

static size_t utf16nbytes(const void *, size_t);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);


archive_string_conversion_charset_name(struct archive_string_conv *sc)
{
	if (sc->flag & SCONV_TO_CHARSET)
		return (sc->to_charset);
	else
		return (sc->from_charset);
}

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);

static struct archive_string_conv *get_sconv_object(struct archive *,
	const char *, const char *, int);

static int archive_wstring_append_from_mbs_in_codepage(
    struct archive_wstring *, const char *, size_t,
    struct archive_string_conv *);

static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
    const wchar_t *, size_t, struct archive_string_conv *);

static int strncat_in_codepage(struct archive_string *, const void *,
    size_t, struct archive_string_conv *);


best_effort_strncat_to_utf16be(struct archive_string *as16, const void *_p,
    size_t length, struct archive_string_conv *sc)
{
	return (best_effort_strncat_to_utf16(as16, _p, length, sc, 1));
}


archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes,
  const char **p)
{
	struct archive_string_conv *sc;
	int r;

	/* If we already have a UTF8 form, return that immediately. */
	if (aes->aes_set & AES_SET_UTF8) {
		*p = aes->aes_utf8.s;
		return (0);
	}

	*p = NULL;
	if (aes->aes_set & AES_SET_MBS) {
		sc = archive_string_conversion_to_charset(a, "UTF-8", 1);
		if (sc == NULL)
			return (-1);/* Couldn't allocate memory for sc. */
		r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s,
		    aes->aes_mbs.length, sc);
		if (a == NULL)
			free_sconv_object(sc);
		if (r == 0) {
			aes->aes_set |= AES_SET_UTF8;
			*p = aes->aes_utf8.s;
			return (0);/* success. */
		} else
			return (-1);/* failure. */
	}
	return (0);/* success. */
}


best_effort_strncat_from_utf16be(struct archive_string *as, const void *_p,
    size_t bytes, struct archive_string_conv *sc)
{
	return (best_effort_strncat_from_utf16(as, _p, bytes, sc, 1));
}


archive_wstring_append_from_mbs(struct archive_wstring *dest,
    const char *p, size_t len)
{
	size_t r;
	int ret_val = 0;
	/*
	 * No single byte will be more than one wide character,
	 * so this length estimate will always be big enough.
	 */
	// size_t wcs_length = len;
	size_t mbs_length = len;
	const char *mbs = p;
	wchar_t *wcs;
#if HAVE_MBRTOWC
	mbstate_t shift_state;

	memset(&shift_state, 0, sizeof(shift_state));
#endif
	/*
	 * As we decided to have wcs_length == mbs_length == len
	 * we can use len here instead of wcs_length
	 */
	if (NULL == archive_wstring_ensure(dest, dest->length + len + 1))
		return (-1);
	wcs = dest->s + dest->length;
	/*
	 * We cannot use mbsrtowcs/mbstowcs here because those may convert
	 * extra MBS when strlen(p) > len and one wide character consists of
	 * multi bytes.
	 */
	while (*mbs && mbs_length > 0) {
		/*
		 * The buffer we allocated is always big enough.
		 * Keep this code path in a comment if we decide to choose
		 * smaller wcs_length in the future
		 */
/*
		if (wcs_length == 0) {
			dest->length = wcs - dest->s;
			dest->s[dest->length] = L'\0';
			wcs_length = mbs_length;
			if (NULL == archive_wstring_ensure(dest,
			    dest->length + wcs_length + 1))
				return (-1);
			wcs = dest->s + dest->length;
		}
*/
#if HAVE_MBRTOWC
		r = mbrtowc(wcs, mbs, mbs_length, &shift_state);
#else
		r = mbtowc(wcs, mbs, mbs_length);
#endif
		if (r == (size_t)-1 || r == (size_t)-2) {
			ret_val = -1;
			break;
		}
		if (r == 0 || r > mbs_length)
			break;
		wcs++;
		// wcs_length--;
		mbs += r;
		mbs_length -= r;
	}
	dest->length = wcs - dest->s;
	dest->s[dest->length] = L'\0';
	return (ret_val);
}

#define wmemcpy(a,b,i)  (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t))

#define wmemmove(a,b,i)  (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t))

struct archive_string_conv {
	struct archive_string_conv	*next;
	char				*from_charset;
	char				*to_charset;
	unsigned			 from_cp;
	unsigned			 to_cp;
	/* Set 1 if from_charset and to_charset are the same. */
	int				 same;
	int				 flag;
#define SCONV_TO_CHARSET	1	/* MBS is being converted to specified
					 * charset. */
#define SCONV_FROM_CHARSET	(1<<1)	/* MBS is being converted from
					 * specified charset. */
#define SCONV_BEST_EFFORT 	(1<<2)	/* Copy at least ASCII code. */
#define SCONV_WIN_CP	 	(1<<3)	/* Use Windows API for converting
					 * MBS. */
#define SCONV_UTF8_LIBARCHIVE_2 (1<<4)	/* Incorrect UTF-8 made by libarchive
					 * 2.x in the wrong assumption. */
#define SCONV_NORMALIZATION_C	(1<<6)	/* Need normalization to be Form C.
					 * Before UTF-8 characters are actually
					 * processed. */
#define SCONV_NORMALIZATION_D	(1<<7)	/* Need normalization to be Form D.
					 * Before UTF-8 characters are actually
					 * processed.
					 * Currently this only for MAC OS X. */
#define SCONV_TO_UTF8		(1<<8)	/* "to charset" side is UTF-8. */
#define SCONV_FROM_UTF8		(1<<9)	/* "from charset" side is UTF-8. */
#define SCONV_TO_UTF16BE 	(1<<10)	/* "to charset" side is UTF-16BE. */
#define SCONV_FROM_UTF16BE 	(1<<11)	/* "from charset" side is UTF-16BE. */
#define SCONV_TO_UTF16LE 	(1<<12)	/* "to charset" side is UTF-16LE. */
#define SCONV_FROM_UTF16LE 	(1<<13)	/* "from charset" side is UTF-16LE. */
#define SCONV_TO_UTF16		(SCONV_TO_UTF16BE | SCONV_TO_UTF16LE)
#define SCONV_FROM_UTF16	(SCONV_FROM_UTF16BE | SCONV_FROM_UTF16LE)

#if HAVE_ICONV
	iconv_t				 cd;
	iconv_t				 cd_w;/* Use at archive_mstring on
				 	       * Windows. */
#endif
	/* A temporary buffer for normalization. */
	struct archive_string		 utftmp;
	int (*converter[2])(struct archive_string *, const void *, size_t,
	    struct archive_string_conv *);
	int				 nconverter;
}

#define CP_C_LOCALE	0	/* "C" locale only for this file. */

#define UNICODE_R_CHAR		0xFFFD	/* Replacement character. */

static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd};

static struct archive_string_conv *find_sconv_object(struct archive *,
	const char *, const char *);

static void add_sconv_object(struct archive *, struct archive_string_conv *);

static struct archive_string_conv *create_sconv_object(const char *,
	const char *, unsigned, int);

static void free_sconv_object(struct archive_string_conv *);


static void queue_release_one_tty(struct kref *kref)
{
	struct tty_struct *tty = container_of(kref, struct tty_struct, kref);

	/* The hangup queue is now free so we can reuse it rather than
	   waste a chunk of memory for each port */
	INIT_WORK(&tty->hangup_work, release_one_tty);
	schedule_work(&tty->hangup_work);
}

tty: Fix ->session locking

Currently, locking of ->session is very inconsistent; most places
protect it using the legacy tty mutex, but disassociate_ctty(),
__do_SAK(), tiocspgrp() and tiocgsid() don't.
Two of the writers hold the ctrl_lock (because they already need it for
->pgrp), but __proc_set_tty() doesn't do that yet.

On a PREEMPT=y system, an unprivileged user can theoretically abuse
this broken locking to read 4 bytes of freed memory via TIOCGSID if
tiocgsid() is preempted long enough at the right point. (Other things
might also go wrong, especially if root-only ioctls are involved; I'm
not sure about that.)

Change the locking on ->session such that:

 - tty_lock() is held by all writers: By making disassociate_ctty()
   hold it. This should be fine because the same lock can already be
   taken through the call to tty_vhangup_session().
   The tricky part is that we need to shorten the area covered by
   siglock to be able to take tty_lock() without ugly retry logic; as
   far as I can tell, this should be fine, since nothing in the
   signal_struct is touched in the `if (tty)` branch.
 - ctrl_lock is held by all writers: By changing __proc_set_tty() to
   hold the lock a little longer.
 - All readers that aren't holding tty_lock() hold ctrl_lock: By
   adding locking to tiocgsid() and __do_SAK(), and expanding the area
   covered by ctrl_lock in tiocspgrp().

Cc: stable@kernel.org
Signed-off-by: Jann Horn <jannh@google.com>
Reviewed-by: Jiri Slaby <jirislaby@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>


void console_sysfs_notify(void)
{
	if (consdev)
		sysfs_notify(&consdev->kobj, NULL, "active");
}


static void tty_del_file(struct file *file)
{
	struct tty_file_private *priv = file->private_data;
	struct tty_struct *tty = priv->tty;

	spin_lock(&tty->files_lock);
	list_del(&priv->list);
	spin_unlock(&tty->files_lock);
	tty_free_file(file);
}

# define tty_debug_hangup(tty, f, args...)	tty_debug(tty, f, ##args)

# define tty_debug_hangup(tty, f, args...)	do { } while (0)

struct ktermios tty_std_termios = {	/* for the benefit of tty drivers  */
	.c_iflag = ICRNL | IXON,
	.c_oflag = OPOST | ONLCR,
	.c_cflag = B38400 | CS8 | CREAD | HUPCL,
	.c_lflag = ISIG | ICANON | ECHO | ECHOE | ECHOK |
		   ECHOCTL | ECHOKE | IEXTEN,
	.c_cc = INIT_C_CC,
	.c_ispeed = 38400,
	.c_ospeed = 38400,
	/* .c_line = N_TTY, */
};

static ssize_t tty_read(struct file *, char __user *, size_t, loff_t *);

static ssize_t tty_write(struct file *, const char __user *, size_t, loff_t *);

ssize_t redirected_tty_write(struct file *, const char __user *,
							size_t, loff_t *);

static __poll_t tty_poll(struct file *, poll_table *);

static int tty_open(struct inode *, struct file *);

long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg);

static long tty_compat_ioctl(struct file *file, unsigned int cmd,
				unsigned long arg);

static int __tty_fasync(int fd, struct file *filp, int on);

static int tty_fasync(int fd, struct file *filp, int on);

static void release_tty(struct tty_struct *tty, int idx);

static const struct file_operations tty_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
	.show_fdinfo	= tty_show_fdinfo,
};

static const struct file_operations console_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= redirected_tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
};

static const struct file_operations hung_up_tty_fops = {
	.llseek		= no_llseek,
	.read		= hung_up_tty_read,
	.write		= hung_up_tty_write,
	.poll		= hung_up_tty_poll,
	.unlocked_ioctl	= hung_up_tty_ioctl,
	.compat_ioctl	= hung_up_tty_compat_ioctl,
	.release	= tty_release,
	.fasync		= hung_up_tty_fasync,
};

static DEFINE_SPINLOCK(redirect_lock);

static struct file *redirect;

extern void tty_sysctl_init(void);

static char ptychar[] = "pqrstuvwxyzabcde";


static int tty_tiocmset(struct tty_struct *tty, unsigned int cmd,
	     unsigned __user *p)
{
	int retval;
	unsigned int set, clear, val;

	if (tty->ops->tiocmset == NULL)
		return -EINVAL;

	retval = get_user(val, p);
	if (retval)
		return retval;
	set = clear = 0;
	switch (cmd) {
	case TIOCMBIS:
		set = val;
		break;
	case TIOCMBIC:
		clear = val;
		break;
	case TIOCMSET:
		set = val;
		clear = ~val;
		break;
	}
	set &= TIOCM_DTR|TIOCM_RTS|TIOCM_OUT1|TIOCM_OUT2|TIOCM_LOOP;
	clear &= TIOCM_DTR|TIOCM_RTS|TIOCM_OUT1|TIOCM_OUT2|TIOCM_LOOP;
	return tty->ops->tiocmset(tty, set, clear);
}


void __start_tty(struct tty_struct *tty)
{
	if (!tty->stopped || tty->flow_stopped)
		return;
	tty->stopped = 0;
	if (tty->ops->start)
		tty->ops->start(tty);
	tty_wakeup(tty);
}


static ssize_t tty_write(struct file *file, const char __user *buf,
						size_t count, loff_t *ppos)
{
	struct tty_struct *tty = file_tty(file);
 	struct tty_ldisc *ld;
	ssize_t ret;

	if (tty_paranoia_check(tty, file_inode(file), "tty_write"))
		return -EIO;
	if (!tty || !tty->ops->write ||	tty_io_error(tty))
			return -EIO;
	/* Short term debug to catch buggy drivers */
	if (tty->ops->write_room == NULL)
		tty_err(tty, "missing write_room method\n");
	ld = tty_ldisc_ref_wait(tty);
	if (!ld)
		return hung_up_tty_write(file, buf, count, ppos);
	if (!ld->ops->write)
		ret = -EIO;
	else
		ret = do_tty_write(ld->ops->write, tty, file, buf, count);
	tty_ldisc_deref(ld);
	return ret;
}

# define tty_debug_hangup(tty, f, args...)	tty_debug(tty, f, ##args)

# define tty_debug_hangup(tty, f, args...)	do { } while (0)

struct ktermios tty_std_termios = {	/* for the benefit of tty drivers  */
	.c_iflag = ICRNL | IXON,
	.c_oflag = OPOST | ONLCR,
	.c_cflag = B38400 | CS8 | CREAD | HUPCL,
	.c_lflag = ISIG | ICANON | ECHO | ECHOE | ECHOK |
		   ECHOCTL | ECHOKE | IEXTEN,
	.c_cc = INIT_C_CC,
	.c_ispeed = 38400,
	.c_ospeed = 38400,
	/* .c_line = N_TTY, */
};

static ssize_t tty_read(struct file *, char __user *, size_t, loff_t *);

static ssize_t tty_write(struct file *, const char __user *, size_t, loff_t *);

ssize_t redirected_tty_write(struct file *, const char __user *,
							size_t, loff_t *);

static __poll_t tty_poll(struct file *, poll_table *);

static int tty_open(struct inode *, struct file *);

long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg);

static long tty_compat_ioctl(struct file *file, unsigned int cmd,
				unsigned long arg);

#define tty_compat_ioctl NULL

static int __tty_fasync(int fd, struct file *filp, int on);

static int tty_fasync(int fd, struct file *filp, int on);

static void release_tty(struct tty_struct *tty, int idx);

static const struct file_operations tty_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
	.show_fdinfo	= tty_show_fdinfo,
};

static const struct file_operations console_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= redirected_tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
};

static const struct file_operations hung_up_tty_fops = {
	.llseek		= no_llseek,
	.read		= hung_up_tty_read,
	.write		= hung_up_tty_write,
	.poll		= hung_up_tty_poll,
	.unlocked_ioctl	= hung_up_tty_ioctl,
	.compat_ioctl	= hung_up_tty_compat_ioctl,
	.release	= tty_release,
	.fasync		= hung_up_tty_fasync,
};

static DEFINE_SPINLOCK(redirect_lock);

static struct file *redirect;

static char ptychar[] = "pqrstuvwxyzabcde";


struct tty_struct *tty_init_dev(struct tty_driver *driver, int idx)
{
	struct tty_struct *tty;
	int retval;

	/*
	 * First time open is complex, especially for PTY devices.
	 * This code guarantees that either everything succeeds and the
	 * TTY is ready for operation, or else the table slots are vacated
	 * and the allocated memory released.  (Except that the termios
	 * may be retained.)
	 */

	if (!try_module_get(driver->owner))
		return ERR_PTR(-ENODEV);

	tty = alloc_tty_struct(driver, idx);
	if (!tty) {
		retval = -ENOMEM;
		goto err_module_put;
	}

	tty_lock(tty);
	retval = tty_driver_install_tty(driver, tty);
	if (retval < 0)
		goto err_free_tty;

	if (!tty->port)
		tty->port = driver->ports[idx];

	if (WARN_RATELIMIT(!tty->port,
			"%s: %s driver does not set tty->port. This would crash the kernel. Fix the driver!\n",
			__func__, tty->driver->name)) {
		retval = -EINVAL;
		goto err_release_lock;
	}

	retval = tty_ldisc_lock(tty, 5 * HZ);
	if (retval)
		goto err_release_lock;
	tty->port->itty = tty;

	/*
	 * Structures all installed ... call the ldisc open routines.
	 * If we fail here just call release_tty to clean up.  No need
	 * to decrement the use counts, as release_tty doesn't care.
	 */
	retval = tty_ldisc_setup(tty, tty->link);
	if (retval)
		goto err_release_tty;
	tty_ldisc_unlock(tty);
	/* Return the tty locked so that it cannot vanish under the caller */
	return tty;

err_free_tty:
	tty_unlock(tty);
	free_tty_struct(tty);
err_module_put:
	module_put(driver->owner);
	return ERR_PTR(retval);

	/* call the tty release_tty routine to clean out this slot */
err_release_tty:
	tty_ldisc_unlock(tty);
	tty_info_ratelimited(tty, "ldisc open failed (%d), clearing slot %d\n",
			     retval, idx);
err_release_lock:
	tty_unlock(tty);
	release_tty(tty, idx);
	return ERR_PTR(retval);
}

# define tty_debug_hangup(tty, f, args...)	tty_debug(tty, f, ##args)

# define tty_debug_hangup(tty, f, args...)	do { } while (0)

struct ktermios tty_std_termios = {	/* for the benefit of tty drivers  */
	.c_iflag = ICRNL | IXON,
	.c_oflag = OPOST | ONLCR,
	.c_cflag = B38400 | CS8 | CREAD | HUPCL,
	.c_lflag = ISIG | ICANON | ECHO | ECHOE | ECHOK |
		   ECHOCTL | ECHOKE | IEXTEN,
	.c_cc = INIT_C_CC,
	.c_ispeed = 38400,
	.c_ospeed = 38400,
	/* .c_line = N_TTY, */
};

static ssize_t tty_read(struct file *, char __user *, size_t, loff_t *);

static ssize_t tty_write(struct file *, const char __user *, size_t, loff_t *);

ssize_t redirected_tty_write(struct file *, const char __user *,
							size_t, loff_t *);

static __poll_t tty_poll(struct file *, poll_table *);

static int tty_open(struct inode *, struct file *);

long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg);

static long tty_compat_ioctl(struct file *file, unsigned int cmd,
				unsigned long arg);

static int __tty_fasync(int fd, struct file *filp, int on);

static int tty_fasync(int fd, struct file *filp, int on);

static void release_tty(struct tty_struct *tty, int idx);

static const struct file_operations tty_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
	.show_fdinfo	= tty_show_fdinfo,
};

static const struct file_operations console_fops = {
	.llseek		= no_llseek,
	.read		= tty_read,
	.write		= redirected_tty_write,
	.poll		= tty_poll,
	.unlocked_ioctl	= tty_ioctl,
	.compat_ioctl	= tty_compat_ioctl,
	.open		= tty_open,
	.release	= tty_release,
	.fasync		= tty_fasync,
};

static const struct file_operations hung_up_tty_fops = {
	.llseek		= no_llseek,
	.read		= hung_up_tty_read,
	.write		= hung_up_tty_write,
	.poll		= hung_up_tty_poll,
	.unlocked_ioctl	= hung_up_tty_ioctl,
	.compat_ioctl	= hung_up_tty_compat_ioctl,
	.release	= tty_release,
	.fasync		= hung_up_tty_fasync,
};

static struct file *redirect;