filemess(
    buf_T	*buf,
    char_u	*name,
    char_u	*s,
    int		attr)
{
    int		msg_scroll_save;

    if (msg_silent != 0)
	return;
    msg_add_fname(buf, name);	    /* put file name in IObuff with quotes */
    /* If it's extremely long, truncate it. */
    if (STRLEN(IObuff) > IOSIZE - 80)
	IObuff[IOSIZE - 80] = NUL;
    STRCAT(IObuff, s);
    /*
     * For the first message may have to start a new line.
     * For further ones overwrite the previous one, reset msg_scroll before
     * calling filemess().
     */
    msg_scroll_save = msg_scroll;
    if (shortmess(SHM_OVERALL) && !exiting && p_verbose == 0)
	msg_scroll = FALSE;
    if (!msg_scroll)	/* wait a bit when overwriting an error msg */
	check_for_delay(FALSE);
    msg_start();
    msg_scroll = msg_scroll_save;
    msg_scrolled_ign = TRUE;
    /* may truncate the message to avoid a hit-return prompt */
    msg_outtrans_attr(msg_may_trunc(FALSE, IObuff), attr);
    msg_clr_eos();
    out_flush();
    msg_scrolled_ign = FALSE;
}

static char_u *next_fenc(char_u **pp);

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

#  define FIO_PUT_CP(x) (((x) & 0xffff) << 16)	/* put codepage in top word */

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static linenr_T readfile_linenr(linenr_T linecnt, char_u *p, char_u *endp);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);

int		read_buffer = (flags & READ_BUFFER);

int		read_fifo = (flags & READ_FIFO);


read_eintr(int fd, void *buf, size_t bufsize)
{
    long ret;

    for (;;)
    {
	ret = vim_read(fd, buf, bufsize);
	if (ret >= 0 || errno != EINTR)
	    break;
    }
    return ret;
}

patch 8.0.1263: others can read the swap file if a user is careless

Problem:    Others can read the swap file if a user is careless with his
            primary group.
Solution:   If the group permission allows for reading but the world
            permissions doesn't, make sure the group is right.

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static void check_marks_read(void);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static void msg_add_eol(void);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);

int		read_buffer = (flags & READ_BUFFER);

int		read_fifo = (flags & READ_FIFO);

int		set_options = newfile || read_buffer
					   || (eap != NULL && eap->read_edit);

int		wasempty;

long	size = 0;

int		skip_read = FALSE;

int		did_ask_for_key = FALSE;

int		read_undo_file = FALSE;

int		split = 0;

int		error = FALSE;


check_for_cryptkey(
    char_u	*cryptkey,	/* previous encryption key or NULL */
    char_u	*ptr,		/* pointer to read bytes */
    long	*sizep,		/* length of read bytes */
    off_T	*filesizep,	/* nr of bytes used from file */
    int		newfile,	/* editing a new buffer */
    char_u	*fname,		/* file name to display */
    int		*did_ask)	/* flag: whether already asked for key */
{
    int method = crypt_method_nr_from_magic((char *)ptr, *sizep);
    int b_p_ro = curbuf->b_p_ro;

    if (method >= 0)
    {
	/* Mark the buffer as read-only until the decryption has taken place.
	 * Avoids accidentally overwriting the file with garbage. */
	curbuf->b_p_ro = TRUE;

	/* Set the cryptmethod local to the buffer. */
	crypt_set_cm_option(curbuf, method);
	if (cryptkey == NULL && !*did_ask)
	{
	    if (*curbuf->b_p_key)
		cryptkey = curbuf->b_p_key;
	    else
	    {
		/* When newfile is TRUE, store the typed key in the 'key'
		 * option and don't free it.  bf needs hash of the key saved.
		 * Don't ask for the key again when first time Enter was hit.
		 * Happens when retrying to detect encoding. */
		smsg((char_u *)_(need_key_msg), fname);
		msg_scroll = TRUE;
		crypt_check_method(method);
		cryptkey = crypt_get_key(newfile, FALSE);
		*did_ask = TRUE;

		/* check if empty key entered */
		if (cryptkey != NULL && *cryptkey == NUL)
		{
		    if (cryptkey != curbuf->b_p_key)
			vim_free(cryptkey);
		    cryptkey = NULL;
		}
	    }
	}

	if (cryptkey != NULL)
	{
	    int header_len;

	    curbuf->b_cryptstate = crypt_create_from_header(
						       method, cryptkey, ptr);
	    crypt_set_cm_option(curbuf, method);

	    /* Remove cryptmethod specific header from the text. */
	    header_len = crypt_get_header_len(method);
	    if (*sizep <= header_len)
		/* invalid header, buffer can't be encrypted */
		return NULL;
	    *filesizep += header_len;
	    *sizep -= header_len;
	    mch_memmove(ptr, ptr + header_len, (size_t)*sizep);

	    /* Restore the read-only flag. */
	    curbuf->b_p_ro = b_p_ro;
	}
    }
    /* When starting to edit a new file which does not have encryption, clear
     * the 'key' option, except when starting up (called with -x argument) */
    else if (newfile && *curbuf->b_p_key != NUL && !starting)
	set_option_value((char_u *)"key", 0L, (char_u *)"", OPT_LOCAL);

    return cryptkey;
}

#define BUFSIZE		8192	/* size of normal write buffer */

#define SMBUFSIZE	256	/* size of emergency write buffer */

static char_u *next_fenc(char_u **pp);

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

#  define FIO_GET_CP(x)	(((x)>>16) & 0xffff)	/* get codepage from top word */

# define FIO_ENCRYPTED	0x1000	/* encrypt written bytes */

# define FIO_NOCONVERT	0x2000	/* skip encoding conversion */

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static linenr_T readfile_linenr(linenr_T linecnt, char_u *p, char_u *endp);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");


vim_rename(char_u *from, char_u *to)
{
    int		fd_in;
    int		fd_out;
    int		n;
    char	*errmsg = NULL;
    char	*buffer;
#ifdef AMIGA
    BPTR	flock;
#endif
    stat_T	st;
    long	perm;
#ifdef HAVE_ACL
    vim_acl_T	acl;		/* ACL from original file */
#endif
    int		use_tmp_file = FALSE;

    /*
     * When the names are identical, there is nothing to do.  When they refer
     * to the same file (ignoring case and slash/backslash differences) but
     * the file name differs we need to go through a temp file.
     */
    if (fnamecmp(from, to) == 0)
    {
	if (p_fic && STRCMP(gettail(from), gettail(to)) != 0)
	    use_tmp_file = TRUE;
	else
	    return 0;
    }

    /*
     * Fail if the "from" file doesn't exist.  Avoids that "to" is deleted.
     */
    if (mch_stat((char *)from, &st) < 0)
	return -1;

#ifdef UNIX
    {
	stat_T	st_to;

	/* It's possible for the source and destination to be the same file.
	 * This happens when "from" and "to" differ in case and are on a FAT32
	 * filesystem.  In that case go through a temp file name. */
	if (mch_stat((char *)to, &st_to) >= 0
		&& st.st_dev == st_to.st_dev
		&& st.st_ino == st_to.st_ino)
	    use_tmp_file = TRUE;
    }
#endif
#ifdef WIN3264
    {
	BY_HANDLE_FILE_INFORMATION info1, info2;

	/* It's possible for the source and destination to be the same file.
	 * In that case go through a temp file name.  This makes rename("foo",
	 * "./foo") a no-op (in a complicated way). */
	if (win32_fileinfo(from, &info1) == FILEINFO_OK
		&& win32_fileinfo(to, &info2) == FILEINFO_OK
		&& info1.dwVolumeSerialNumber == info2.dwVolumeSerialNumber
		&& info1.nFileIndexHigh == info2.nFileIndexHigh
		&& info1.nFileIndexLow == info2.nFileIndexLow)
	    use_tmp_file = TRUE;
    }
#endif

    if (use_tmp_file)
    {
	char	tempname[MAXPATHL + 1];

	/*
	 * Find a name that doesn't exist and is in the same directory.
	 * Rename "from" to "tempname" and then rename "tempname" to "to".
	 */
	if (STRLEN(from) >= MAXPATHL - 5)
	    return -1;
	STRCPY(tempname, from);
	for (n = 123; n < 99999; ++n)
	{
	    sprintf((char *)gettail((char_u *)tempname), "%d", n);
	    if (mch_stat(tempname, &st) < 0)
	    {
		if (mch_rename((char *)from, tempname) == 0)
		{
		    if (mch_rename(tempname, (char *)to) == 0)
			return 0;
		    /* Strange, the second step failed.  Try moving the
		     * file back and return failure. */
		    mch_rename(tempname, (char *)from);
		    return -1;
		}
		/* If it fails for one temp name it will most likely fail
		 * for any temp name, give up. */
		return -1;
	    }
	}
	return -1;
    }

    /*
     * Delete the "to" file, this is required on some systems to make the
     * mch_rename() work, on other systems it makes sure that we don't have
     * two files when the mch_rename() fails.
     */

#ifdef AMIGA
    /*
     * With MSDOS-compatible filesystems (crossdos, messydos) it is possible
     * that the name of the "to" file is the same as the "from" file, even
     * though the names are different. To avoid the chance of accidentally
     * deleting the "from" file (horror!) we lock it during the remove.
     *
     * When used for making a backup before writing the file: This should not
     * happen with ":w", because startscript() should detect this problem and
     * set buf->b_shortname, causing modname() to return a correct ".bak" file
     * name.  This problem does exist with ":w filename", but then the
     * original file will be somewhere else so the backup isn't really
     * important. If autoscripting is off the rename may fail.
     */
    flock = Lock((UBYTE *)from, (long)ACCESS_READ);
#endif
    mch_remove(to);
#ifdef AMIGA
    if (flock)
	UnLock(flock);
#endif

    /*
     * First try a normal rename, return if it works.
     */
    if (mch_rename((char *)from, (char *)to) == 0)
	return 0;

    /*
     * Rename() failed, try copying the file.
     */
    perm = mch_getperm(from);
#ifdef HAVE_ACL
    /* For systems that support ACL: get the ACL from the original file. */
    acl = mch_get_acl(from);
#endif
    fd_in = mch_open((char *)from, O_RDONLY|O_EXTRA, 0);
    if (fd_in == -1)
    {
#ifdef HAVE_ACL
	mch_free_acl(acl);
#endif
	return -1;
    }

    /* Create the new file with same permissions as the original. */
    fd_out = mch_open((char *)to,
		       O_CREAT|O_EXCL|O_WRONLY|O_EXTRA|O_NOFOLLOW, (int)perm);
    if (fd_out == -1)
    {
	close(fd_in);
#ifdef HAVE_ACL
	mch_free_acl(acl);
#endif
	return -1;
    }

    buffer = (char *)alloc(BUFSIZE);
    if (buffer == NULL)
    {
	close(fd_out);
	close(fd_in);
#ifdef HAVE_ACL
	mch_free_acl(acl);
#endif
	return -1;
    }

    while ((n = read_eintr(fd_in, buffer, BUFSIZE)) > 0)
	if (write_eintr(fd_out, buffer, n) != n)
	{
	    errmsg = _("E208: Error writing to \"%s\"");
	    break;
	}

    vim_free(buffer);
    close(fd_in);
    if (close(fd_out) < 0)
	errmsg = _("E209: Error closing \"%s\"");
    if (n < 0)
    {
	errmsg = _("E210: Error reading \"%s\"");
	to = from;
    }
#ifndef UNIX	    /* for Unix mch_open() already set the permission */
    mch_setperm(to, perm);
#endif
#ifdef HAVE_ACL
    mch_set_acl(to, acl);
    mch_free_acl(acl);
#endif
#if defined(HAVE_SELINUX) || defined(HAVE_SMACK)
    mch_copy_sec(from, to);
#endif
    if (errmsg != NULL)
    {
	EMSG2(errmsg, to);
	return -1;
    }
    mch_remove(from);
    return 0;
}

#define BUFSIZE		8192	/* size of normal write buffer */

#define SMBUFSIZE	256	/* size of emergency write buffer */

static char_u *next_fenc(char_u **pp);

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

#  define FIO_GET_CP(x)	(((x)>>16) & 0xffff)	/* get codepage from top word */

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static linenr_T readfile_linenr(linenr_T linecnt, char_u *p, char_u *endp);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);


find_end_event(
    char_u  *arg,
    int	    have_group)	    /* TRUE when group name was found */
{
    char_u  *pat;
    char_u  *p;

    if (*arg == '*')
    {
	if (arg[1] && !VIM_ISWHITE(arg[1]))
	{
	    EMSG2(_("E215: Illegal character after *: %s"), arg);
	    return NULL;
	}
	pat = arg + 1;
    }
    else
    {
	for (pat = arg; *pat && *pat != '|' && !VIM_ISWHITE(*pat); pat = p)
	{
	    if ((int)event_name2nr(pat, &p) >= (int)NUM_EVENTS)
	    {
		if (have_group)
		    EMSG2(_("E216: No such event: %s"), pat);
		else
		    EMSG2(_("E216: No such group or event: %s"), pat);
		return NULL;
	    }
	}
    }
    return pat;
}

static char_u *next_fenc(char_u **pp);

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

# define AUGROUP_DEFAULT    -1	    /* default autocmd group */

# define AUGROUP_ERROR	    -2	    /* erroneous autocmd group */

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static linenr_T readfile_linenr(linenr_T linecnt, char_u *p, char_u *endp);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);

int		read_buffer = (flags & READ_BUFFER);

int		read_fifo = (flags & READ_FIFO);


readfile_charconvert(
    char_u	*fname,		/* name of input file */
    char_u	*fenc,		/* converted from */
    int		*fdp)		/* in/out: file descriptor of file */
{
    char_u	*tmpname;
    char_u	*errmsg = NULL;

    tmpname = vim_tempname('r', FALSE);
    if (tmpname == NULL)
	errmsg = (char_u *)_("Can't find temp file for conversion");
    else
    {
	close(*fdp);		/* close the input file, ignore errors */
	*fdp = -1;
	if (eval_charconvert(fenc, enc_utf8 ? (char_u *)"utf-8" : p_enc,
						      fname, tmpname) == FAIL)
	    errmsg = (char_u *)_("Conversion with 'charconvert' failed");
	if (errmsg == NULL && (*fdp = mch_open((char *)tmpname,
						  O_RDONLY | O_EXTRA, 0)) < 0)
	    errmsg = (char_u *)_("can't read output of 'charconvert'");
    }

    if (errmsg != NULL)
    {
	/* Don't use emsg(), it breaks mappings, the retry with
	 * another type of conversion might still work. */
	MSG(errmsg);
	if (tmpname != NULL)
	{
	    mch_remove(tmpname);	/* delete converted file */
	    vim_free(tmpname);
	    tmpname = NULL;
	}
    }

    /* If the input file is closed, open it (caller should check for error). */
    if (*fdp < 0)
	*fdp = mch_open((char *)fname, O_RDONLY | O_EXTRA, 0);

    return tmpname;
}

static char_u *next_fenc(char_u **pp);

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

#  define FIO_GET_CP(x)	(((x)>>16) & 0xffff)	/* get codepage from top word */

# define FIO_NOCONVERT	0x2000	/* skip encoding conversion */

# define FIO_UCSBOM	0x4000	/* check for BOM at start of file */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static linenr_T readfile_linenr(linenr_T linecnt, char_u *p, char_u *endp);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);


aucmd_restbuf(
    aco_save_T	*aco)		/* structure holding saved values */
{
    int dummy;

    if (aco->use_aucmd_win)
    {
	--curbuf->b_nwindows;
	/* Find "aucmd_win", it can't be closed, but it may be in another tab
	 * page. Do not trigger autocommands here. */
	block_autocmds();
	if (curwin != aucmd_win)
	{
	    tabpage_T	*tp;
	    win_T	*wp;

	    FOR_ALL_TAB_WINDOWS(tp, wp)
	    {
		if (wp == aucmd_win)
		{
		    if (tp != curtab)
			goto_tabpage_tp(tp, TRUE, TRUE);
		    win_goto(aucmd_win);
		    goto win_found;
		}
	    }
	}
win_found:

	/* Remove the window and frame from the tree of frames. */
	(void)winframe_remove(curwin, &dummy, NULL);
	win_remove(curwin, NULL);
	aucmd_win_used = FALSE;
	last_status(FALSE);	    /* may need to remove last status line */

	if (!valid_tabpage_win(curtab))
	    /* no valid window in current tabpage */
	    close_tabpage(curtab);

	restore_snapshot(SNAP_AUCMD_IDX, FALSE);
	(void)win_comp_pos();   /* recompute window positions */
	unblock_autocmds();

	if (win_valid(aco->save_curwin))
	    curwin = aco->save_curwin;
	else
	    /* Hmm, original window disappeared.  Just use the first one. */
	    curwin = firstwin;
#ifdef FEAT_EVAL
	vars_clear(&aucmd_win->w_vars->dv_hashtab);  /* free all w: variables */
	hash_init(&aucmd_win->w_vars->dv_hashtab);   /* re-use the hashtab */
#endif
	curbuf = curwin->w_buffer;

	vim_free(globaldir);
	globaldir = aco->globaldir;

	/* the buffer contents may have changed */
	check_cursor();
	if (curwin->w_topline > curbuf->b_ml.ml_line_count)
	{
	    curwin->w_topline = curbuf->b_ml.ml_line_count;
#ifdef FEAT_DIFF
	    curwin->w_topfill = 0;
#endif
	}
#if defined(FEAT_GUI)
	/* Hide the scrollbars from the aucmd_win and update. */
	gui_mch_enable_scrollbar(&aucmd_win->w_scrollbars[SBAR_LEFT], FALSE);
	gui_mch_enable_scrollbar(&aucmd_win->w_scrollbars[SBAR_RIGHT], FALSE);
	gui_may_update_scrollbars();
#endif
    }
    else
    {
	/* restore curwin */
	if (win_valid(aco->save_curwin))
	{
	    /* Restore the buffer which was previously edited by curwin, if
	     * it was changed, we are still the same window and the buffer is
	     * valid. */
	    if (curwin == aco->new_curwin
		    && curbuf != aco->new_curbuf.br_buf
		    && bufref_valid(&aco->new_curbuf)
		    && aco->new_curbuf.br_buf->b_ml.ml_mfp != NULL)
	    {
# if defined(FEAT_SYN_HL) || defined(FEAT_SPELL)
		if (curwin->w_s == &curbuf->b_s)
		    curwin->w_s = &aco->new_curbuf.br_buf->b_s;
# endif
		--curbuf->b_nwindows;
		curbuf = aco->new_curbuf.br_buf;
		curwin->w_buffer = curbuf;
		++curbuf->b_nwindows;
	    }

	    curwin = aco->save_curwin;
	    curbuf = curwin->w_buffer;
	    /* In case the autocommand move the cursor to a position that that
	     * not exist in curbuf. */
	    check_cursor();
	}
    }
}

#define BUFSIZE		8192	/* size of normal write buffer */

#define SMBUFSIZE	256	/* size of emergency write buffer */

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static void check_marks_read(void);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static void msg_add_eol(void);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

# define AUGROUP_ALL	    -3	    /* all autocmd groups */

#  define FIO_GET_CP(x)	(((x)>>16) & 0xffff)	/* get codepage from top word */

# define FIO_ALL	-1	/* allow all formats */

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static char *e_auchangedbuf = N_("E812: Autocommands changed buffer or buffer name");

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);

int		read_buffer = (flags & READ_BUFFER);


has_cursorhold(void)
{
    return (first_autopat[(int)(get_real_state() == NORMAL_BUSY
			    ? EVENT_CURSORHOLD : EVENT_CURSORHOLDI)] != NULL);
}

static char_u *readfile_charconvert(char_u *fname, char_u *fenc, int *fdp);

static void check_marks_read(void);

static char_u *check_for_cryptkey(char_u *cryptkey, char_u *ptr, long *sizep, off_T *filesizep, int newfile, char_u *fname, int *did_ask);

static void set_file_time(char_u *fname, time_t atime, time_t mtime);

static int set_rw_fname(char_u *fname, char_u *sfname);

static int msg_add_fileformat(int eol_type);

static void msg_add_eol(void);

static int check_mtime(buf_T *buf, stat_T *s);

static int time_differs(long t1, long t2);

static int apply_autocmds_exarg(event_T event, char_u *fname, char_u *fname_io, int force, buf_T *buf, exarg_T *eap);

static int au_find_group(char_u *name);

struct bw_info
{
    int		bw_fd;		/* file descriptor */
    char_u	*bw_buf;	/* buffer with data to be written */
    int		bw_len;		/* length of data */
#ifdef HAS_BW_FLAGS
    int		bw_flags;	/* FIO_ flags */
#endif
#ifdef FEAT_CRYPT
    buf_T	*bw_buffer;	/* buffer being written */
#endif
#ifdef FEAT_MBYTE
    char_u	bw_rest[CONV_RESTLEN]; /* not converted bytes */
    int		bw_restlen;	/* nr of bytes in bw_rest[] */
    int		bw_first;	/* first write call */
    char_u	*bw_conv_buf;	/* buffer for writing converted chars */
    int		bw_conv_buflen; /* size of bw_conv_buf */
    int		bw_conv_error;	/* set for conversion error */
    linenr_T	bw_conv_error_lnum;  /* first line with error or zero */
    linenr_T	bw_start_lnum;  /* line number at start of buffer */
# ifdef USE_ICONV
    iconv_t	bw_iconv_fd;	/* descriptor for iconv() or -1 */
# endif
#endif
}

static int  buf_write_bytes(struct bw_info *ip);

static int ucs2bytes(unsigned c, char_u **pp, int flags);

static int need_conversion(char_u *fenc);

static int get_fio_flags(char_u *ptr);

static char_u *check_for_bom(char_u *p, long size, int *lenp, int flags);

static int make_bom(char_u *buf, char_u *name);

static int get_win_fio_flags(char_u *ptr);

static int get_mac_fio_flags(char_u *ptr);

static int move_lines(buf_T *frombuf, buf_T *tobuf);

static void vim_settempdir(char_u *tempdir);

static int au_did_filetype

int		fd = 0;

int		newfile = (flags & READ_NEW);

int		check_readonly;

int		filtering = (flags & READ_FILTER);

int		read_stdin = (flags & READ_STDIN);

int		read_buffer = (flags & READ_BUFFER);

int		read_fifo = (flags & READ_FIFO);

int		set_options = newfile || read_buffer
					   || (eap != NULL && eap->read_edit);

char_u	*ptr = NULL;

char_u	*buffer = NULL;

char_u	*new_buffer = NULL;

char_u	*line_start = NULL;

int		wasempty;

int		skip_read = FALSE;

char_u	*cryptkey = NULL;

int		did_ask_for_key = FALSE;

int		read_undo_file = FALSE;


static void jit_fill_hole(void *area, unsigned int size)
{
	/* Fill whole space with INT3 instructions */
	memset(area, 0xcc, size);
}

bpf, x86: Validate computation of branch displacements for x86-64

The branch displacement logic in the BPF JIT compilers for x86 assumes
that, for any generated branch instruction, the distance cannot
increase between optimization passes.

But this assumption can be violated due to how the distances are
computed. Specifically, whenever a backward branch is processed in
do_jit(), the distance is computed by subtracting the positions in the
machine code from different optimization passes. This is because part
of addrs[] is already updated for the current optimization pass, before
the branch instruction is visited.

And so the optimizer can expand blocks of machine code in some cases.

This can confuse the optimizer logic, where it assumes that a fixed
point has been reached for all machine code blocks once the total
program size stops changing. And then the JIT compiler can output
abnormal machine code containing incorrect branch displacements.

To mitigate this issue, we assert that a fixed point is reached while
populating the output image. This rejects any problematic programs.
The issue affects both x86-32 and x86-64. We mitigate separately to
ease backporting.

Signed-off-by: Piotr Krysiuk <piotras@gmail.com>
Reviewed-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
		      struct bpf_tramp_progs *tp, int stack_size)
{
	int i;
	u8 *prog = *pprog;

	for (i = 0; i < tp->nr_progs; i++) {
		if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size, false))
			return -EINVAL;
	}
	*pprog = prog;
	return 0;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static void emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used,
					u32 stack_depth)
{
	int tcc_off = -4 - round_up(stack_depth, 8);
	u8 *prog = *pprog;
	int pop_bytes = 0;
	int off1 = 42;
	int off2 = 31;
	int off3 = 9;
	int cnt = 0;

	/* count the additional bytes used for popping callee regs from stack
	 * that need to be taken into account for each of the offsets that
	 * are used for bailing out of the tail call
	 */
	pop_bytes = get_pop_bytes(callee_regs_used);
	off1 += pop_bytes;
	off2 += pop_bytes;
	off3 += pop_bytes;

	if (stack_depth) {
		off1 += 7;
		off2 += 7;
		off3 += 7;
	}

	/*
	 * rdi - pointer to ctx
	 * rsi - pointer to bpf_array
	 * rdx - index in bpf_array
	 */

	/*
	 * if (index >= array->map.max_entries)
	 *	goto out;
	 */
	EMIT2(0x89, 0xD2);                        /* mov edx, edx */
	EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
	      offsetof(struct bpf_array, map.max_entries));
#define OFFSET1 (off1 + RETPOLINE_RCX_BPF_JIT_SIZE) /* Number of bytes to jump */
	EMIT2(X86_JBE, OFFSET1);                  /* jbe out */

	/*
	 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
	 *	goto out;
	 */
	EMIT2_off32(0x8B, 0x85, tcc_off);         /* mov eax, dword ptr [rbp - tcc_off] */
	EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
#define OFFSET2 (off2 + RETPOLINE_RCX_BPF_JIT_SIZE)
	EMIT2(X86_JA, OFFSET2);                   /* ja out */
	EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
	EMIT2_off32(0x89, 0x85, tcc_off);         /* mov dword ptr [rbp - tcc_off], eax */

	/* prog = array->ptrs[index]; */
	EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6,       /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */
		    offsetof(struct bpf_array, ptrs));

	/*
	 * if (prog == NULL)
	 *	goto out;
	 */
	EMIT3(0x48, 0x85, 0xC9);                  /* test rcx,rcx */
#define OFFSET3 (off3 + RETPOLINE_RCX_BPF_JIT_SIZE)
	EMIT2(X86_JE, OFFSET3);                   /* je out */

	*pprog = prog;
	pop_callee_regs(pprog, callee_regs_used);
	prog = *pprog;

	EMIT1(0x58);                              /* pop rax */
	if (stack_depth)
		EMIT3_off32(0x48, 0x81, 0xC4,     /* add rsp, sd */
			    round_up(stack_depth, 8));

	/* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */
	EMIT4(0x48, 0x8B, 0x49,                   /* mov rcx, qword ptr [rcx + 32] */
	      offsetof(struct bpf_prog, bpf_func));
	EMIT4(0x48, 0x83, 0xC1,                   /* add rcx, X86_TAIL_CALL_OFFSET */
	      X86_TAIL_CALL_OFFSET);
	/*
	 * Now we're ready to jump into next BPF program
	 * rdi == ctx (1st arg)
	 * rcx == prog->bpf_func + X86_TAIL_CALL_OFFSET
	 */
	RETPOLINE_RCX_BPF_JIT();

	/* out: */
	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)

#define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

#define X86_JB  0x72

#define X86_JAE 0x73

#define X86_JE  0x74

#define X86_JNE 0x75

#define X86_JBE 0x76

#define X86_JA  0x77

#define X86_JL  0x7C

#define X86_JGE 0x7D

#define X86_JLE 0x7E

#define X86_JG  0x7F

#define AUX_REG (MAX_BPF_JIT_REG + 1)

#define X86_REG_R9 (MAX_BPF_JIT_REG + 2)


static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
			 int stack_size)
{
	int i;

	/* Restore function arguments from stack.
	 * For a function that accepts two pointers the sequence will be:
	 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10]
	 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8]
	 */
	for (i = 0; i < min(nr_args, 6); i++)
		emit_ldx(prog, bytes_to_bpf_size(m->arg_size[i]),
			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
			 BPF_REG_FP,
			 -(stack_size - i * 8));
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define X86_REG_R9 (MAX_BPF_JIT_REG + 2)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
			   struct bpf_prog *p, int stack_size, bool mod_ret)
{
	u8 *prog = *pprog;
	u8 *jmp_insn;
	int cnt = 0;

	/* arg1: mov rdi, progs[i] */
	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
	if (emit_call(&prog,
		      p->aux->sleepable ? __bpf_prog_enter_sleepable :
		      __bpf_prog_enter, prog))
			return -EINVAL;
	/* remember prog start time returned by __bpf_prog_enter */
	emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);

	/* if (__bpf_prog_enter*(prog) == 0)
	 *	goto skip_exec_of_prog;
	 */
	EMIT3(0x48, 0x85, 0xC0);  /* test rax,rax */
	/* emit 2 nops that will be replaced with JE insn */
	jmp_insn = prog;
	emit_nops(&prog, 2);

	/* arg1: lea rdi, [rbp - stack_size] */
	EMIT4(0x48, 0x8D, 0x7D, -stack_size);
	/* arg2: progs[i]->insnsi for interpreter */
	if (!p->jited)
		emit_mov_imm64(&prog, BPF_REG_2,
			       (long) p->insnsi >> 32,
			       (u32) (long) p->insnsi);
	/* call JITed bpf program or interpreter */
	if (emit_call(&prog, p->bpf_func, prog))
		return -EINVAL;

	/* BPF_TRAMP_MODIFY_RETURN trampolines can modify the return
	 * of the previous call which is then passed on the stack to
	 * the next BPF program.
	 */
	if (mod_ret)
		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);

	/* replace 2 nops with JE insn, since jmp target is known */
	jmp_insn[0] = X86_JE;
	jmp_insn[1] = prog - jmp_insn - 2;

	/* arg1: mov rdi, progs[i] */
	emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
	/* arg2: mov rsi, rbx <- start time in nsec */
	emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
	if (emit_call(&prog,
		      p->aux->sleepable ? __bpf_prog_exit_sleepable :
		      __bpf_prog_exit, prog))
			return -EINVAL;

	*pprog = prog;
	return 0;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)

#define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

#define X86_JE  0x74

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};


static int emit_nops(u8 **pprog, int len)
{
	u8 *prog = *pprog;
	int i, noplen, cnt = 0;

	while (len > 0) {
		noplen = len;

		if (noplen > ASM_NOP_MAX)
			noplen = ASM_NOP_MAX;

		for (i = 0; i < noplen; i++)
			EMIT1(ideal_nops[noplen][i]);
		len -= noplen;
	}

	*pprog = prog;

	return cnt;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};


static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg)
{
	u8 *prog = *pprog;
	int cnt = 0;

	if (is64) {
		/* mov dst, src */
		EMIT_mov(dst_reg, src_reg);
	} else {
		/* mov32 dst, src */
		if (is_ereg(dst_reg) || is_ereg(src_reg))
			EMIT1(add_2mod(0x40, dst_reg, src_reg));
		EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
	}

	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))


static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
{
	u8 *jg_reloc, *prog = *pprog;
	int pivot, err, jg_bytes = 1, cnt = 0;
	s64 jg_offset;

	if (a == b) {
		/* Leaf node of recursion, i.e. not a range of indices
		 * anymore.
		 */
		EMIT1(add_1mod(0x48, BPF_REG_3));	/* cmp rdx,func */
		if (!is_simm32(progs[a]))
			return -1;
		EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3),
			    progs[a]);
		err = emit_cond_near_jump(&prog,	/* je func */
					  (void *)progs[a], prog,
					  X86_JE);
		if (err)
			return err;

		err = emit_fallback_jump(&prog);	/* jmp thunk/indirect */
		if (err)
			return err;

		*pprog = prog;
		return 0;
	}

	/* Not a leaf node, so we pivot, and recursively descend into
	 * the lower and upper ranges.
	 */
	pivot = (b - a) / 2;
	EMIT1(add_1mod(0x48, BPF_REG_3));		/* cmp rdx,func */
	if (!is_simm32(progs[a + pivot]))
		return -1;
	EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a + pivot]);

	if (pivot > 2) {				/* jg upper_part */
		/* Require near jump. */
		jg_bytes = 4;
		EMIT2_off32(0x0F, X86_JG + 0x10, 0);
	} else {
		EMIT2(X86_JG, 0);
	}
	jg_reloc = prog;

	err = emit_bpf_dispatcher(&prog, a, a + pivot,	/* emit lower_part */
				  progs);
	if (err)
		return err;

	/* From Intel 64 and IA-32 Architectures Optimization
	 * Reference Manual, 3.4.1.4 Code Alignment, Assembly/Compiler
	 * Coding Rule 11: All branch targets should be 16-byte
	 * aligned.
	 */
	emit_align(&prog, 16);
	jg_offset = prog - jg_reloc;
	emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes);

	err = emit_bpf_dispatcher(&prog, a + pivot + 1,	/* emit upper_part */
				  b, progs);
	if (err)
		return err;

	*pprog = prog;
	return 0;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define X86_JE  0x74

#define X86_JG  0x7F

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static bool is_ereg_8l(u32 reg)
{
	return is_ereg(reg) ||
	    (1 << reg) & (BIT(BPF_REG_1) |
			  BIT(BPF_REG_2) |
			  BIT(BPF_REG_FP));
}

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};


static void push_callee_regs(u8 **pprog, bool *callee_regs_used)
{
	u8 *prog = *pprog;
	int cnt = 0;

	if (callee_regs_used[0])
		EMIT1(0x53);         /* push rbx */
	if (callee_regs_used[1])
		EMIT2(0x41, 0x55);   /* push r13 */
	if (callee_regs_used[2])
		EMIT2(0x41, 0x56);   /* push r14 */
	if (callee_regs_used[3])
		EMIT2(0x41, 0x57);   /* push r15 */
	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
				void *old_addr, void *new_addr,
				const bool text_live)
{
	const u8 *nop_insn = ideal_nops[NOP_ATOMIC5];
	u8 old_insn[X86_PATCH_SIZE];
	u8 new_insn[X86_PATCH_SIZE];
	u8 *prog;
	int ret;

	memcpy(old_insn, nop_insn, X86_PATCH_SIZE);
	if (old_addr) {
		prog = old_insn;
		ret = t == BPF_MOD_CALL ?
		      emit_call(&prog, old_addr, ip) :
		      emit_jump(&prog, old_addr, ip);
		if (ret)
			return ret;
	}

	memcpy(new_insn, nop_insn, X86_PATCH_SIZE);
	if (new_addr) {
		prog = new_insn;
		ret = t == BPF_MOD_CALL ?
		      emit_call(&prog, new_addr, ip) :
		      emit_jump(&prog, new_addr, ip);
		if (ret)
			return ret;
	}

	ret = -EBUSY;
	mutex_lock(&text_mutex);
	if (memcmp(ip, old_insn, X86_PATCH_SIZE))
		goto out;
	ret = 1;
	if (memcmp(ip, new_insn, X86_PATCH_SIZE)) {
		if (text_live)
			text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL);
		else
			memcpy(ip, new_insn, X86_PATCH_SIZE);
		ret = 0;
	}
out:
	mutex_unlock(&text_mutex);
	return ret;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define X86_PATCH_SIZE		5

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
{
	u8 *prog = *pprog;
	int cnt = 0;

	switch (size) {
	case BPF_B:
		/* Emit 'movzx rax, byte ptr [rax + off]' */
		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
		break;
	case BPF_H:
		/* Emit 'movzx rax, word ptr [rax + off]' */
		EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
		break;
	case BPF_W:
		/* Emit 'mov eax, dword ptr [rax+0x14]' */
		if (is_ereg(dst_reg) || is_ereg(src_reg))
			EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
		else
			EMIT1(0x8B);
		break;
	case BPF_DW:
		/* Emit 'mov rax, qword ptr [rax+0x14]' */
		EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
		break;
	}
	emit_insn_suffix(&prog, src_reg, dst_reg, off);
	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};


static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
		      int stack_size)
{
	int i;
	/* Store function arguments to stack.
	 * For a function that accepts two pointers the sequence will be:
	 * mov QWORD PTR [rbp-0x10],rdi
	 * mov QWORD PTR [rbp-0x8],rsi
	 */
	for (i = 0; i < min(nr_args, 6); i++)
		emit_stx(prog, bytes_to_bpf_size(m->arg_size[i]),
			 BPF_REG_FP,
			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
			 -(stack_size - i * 8));
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define X86_REG_R9 (MAX_BPF_JIT_REG + 2)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
{
	u8 *prog = *pprog;
	int cnt = 0;
	s64 offset;

	offset = func - (ip + 2 + 4);
	if (!is_simm32(offset)) {
		pr_err("Target %p is out of range\n", func);
		return -EINVAL;
	}
	EMIT2_off32(0x0F, jmp_cond + 0x10, offset);
	*pprog = prog;
	return 0;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
{
	u8 *prog = image;

	sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL);
	return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs);
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static void emit_mov_imm32(u8 **pprog, bool sign_propagate,
			   u32 dst_reg, const u32 imm32)
{
	u8 *prog = *pprog;
	u8 b1, b2, b3;
	int cnt = 0;

	/*
	 * Optimization: if imm32 is positive, use 'mov %eax, imm32'
	 * (which zero-extends imm32) to save 2 bytes.
	 */
	if (sign_propagate && (s32)imm32 < 0) {
		/* 'mov %rax, imm32' sign extends imm32 */
		b1 = add_1mod(0x48, dst_reg);
		b2 = 0xC7;
		b3 = 0xC0;
		EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
		goto done;
	}

	/*
	 * Optimization: if imm32 is zero, use 'xor %eax, %eax'
	 * to save 3 bytes.
	 */
	if (imm32 == 0) {
		if (is_ereg(dst_reg))
			EMIT1(add_2mod(0x40, dst_reg, dst_reg));
		b2 = 0x31; /* xor */
		b3 = 0xC0;
		EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
		goto done;
	}

	/* mov %eax, imm32 */
	if (is_ereg(dst_reg))
		EMIT1(add_1mod(0x40, dst_reg));
	EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
done:
	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT1(b1)		EMIT(b1, 1)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};


static int get_pop_bytes(bool *callee_regs_used)
{
	int bytes = 0;

	if (callee_regs_used[3])
		bytes += 2;
	if (callee_regs_used[2])
		bytes += 2;
	if (callee_regs_used[1])
		bytes += 2;
	if (callee_regs_used[0])
		bytes += 1;

	return bytes;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int cmp_ips(const void *a, const void *b)
{
	const s64 *ipa = a;
	const s64 *ipb = b;

	if (*ipa > *ipb)
		return 1;
	if (*ipa < *ipb)
		return -1;
	return 0;
}

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};

static u8 simple_alu_opcodes[] = {
	[BPF_ADD] = 0x01,
	[BPF_SUB] = 0x29,
	[BPF_AND] = 0x21,
	[BPF_OR] = 0x09,
	[BPF_XOR] = 0x31,
	[BPF_LSH] = 0xE0,
	[BPF_RSH] = 0xE8,
	[BPF_ARSH] = 0xF8,
};

struct jit_context {
	int cleanup_addr; /* Epilogue code offset */
}

struct x64_jit_data {
	struct bpf_binary_header *header;
	int *addrs;
	u8 *image;
	int proglen;
	struct jit_context ctx;
}


static int emit_call(u8 **pprog, void *func, void *ip)
{
	return emit_patch(pprog, func, ip, 0xE8);
}


static bool is_axreg(u32 reg)
{
	return reg == BPF_REG_0;
}


static void emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off)
{
	u8 *prog = *pprog;
	int cnt = 0;

	if (is_imm8(off)) {
		/* 1-byte signed displacement.
		 *
		 * If off == 0 we could skip this and save one extra byte, but
		 * special case of x86 R13 which always needs an offset is not
		 * worth the hassle
		 */
		EMIT2(add_2reg(0x40, ptr_reg, val_reg), off);
	} else {
		/* 4-byte signed displacement */
		EMIT1_off32(add_2reg(0x80, ptr_reg, val_reg), off);
	}
	*pprog = prog;
}

#define EMIT(bytes, len) \
	do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)

#define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)

#define EMIT1_off32(b1, off) \
	do { EMIT1(b1); EMIT(off, 4); } while (0)

#define EMIT2_off32(b1, b2, off) \
	do { EMIT2(b1, b2); EMIT(off, 4); } while (0)

#define EMIT3_off32(b1, b2, b3, off) \
	do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)

#define EMIT4_off32(b1, b2, b3, b4, off) \
	do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)

#define EMIT_mov(DST, SRC)								 \
	do {										 \
		if (DST != SRC)								 \
			EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
	} while (0)

static const int reg2hex[] = {
	[BPF_REG_0] = 0,  /* RAX */
	[BPF_REG_1] = 7,  /* RDI */
	[BPF_REG_2] = 6,  /* RSI */
	[BPF_REG_3] = 2,  /* RDX */
	[BPF_REG_4] = 1,  /* RCX */
	[BPF_REG_5] = 0,  /* R8  */
	[BPF_REG_6] = 3,  /* RBX callee saved */
	[BPF_REG_7] = 5,  /* R13 callee saved */
	[BPF_REG_8] = 6,  /* R14 callee saved */
	[BPF_REG_9] = 7,  /* R15 callee saved */
	[BPF_REG_FP] = 5, /* RBP readonly */
	[BPF_REG_AX] = 2, /* R10 temp register */
	[AUX_REG] = 3,    /* R11 temp register */
	[X86_REG_R9] = 1, /* R9 register, 6th function argument */
};

static const int reg2pt_regs[] = {
	[BPF_REG_0] = offsetof(struct pt_regs, ax),
	[BPF_REG_1] = offsetof(struct pt_regs, di),
	[BPF_REG_2] = offsetof(struct pt_regs, si),
	[BPF_REG_3] = offsetof(struct pt_regs, dx),
	[BPF_REG_4] = offsetof(struct pt_regs, cx),
	[BPF_REG_5] = offsetof(struct pt_regs, r8),
	[BPF_REG_6] = offsetof(struct pt_regs, bx),
	[BPF_REG_7] = offsetof(struct pt_regs, r13),
	[BPF_REG_8] = offsetof(struct pt_regs, r14),
	[BPF_REG_9] = offsetof(struct pt_regs, r15),
};