int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
{
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;

	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

	memcg_propagate_slab_attrs(s);
	err = sysfs_slab_add(s);
	if (err)
		__kmem_cache_release(s);

	return err;
}

#define __OBJECT_POISON		((slab_flags_t __force)0x80000000U)

#define __CMPXCHG_DOUBLE	((slab_flags_t __force)0x40000000U)

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
{
	return atomic_long_read(&n->nr_slabs);
}

mm: slub: add missing TID bump in kmem_cache_alloc_bulk()

When kmem_cache_alloc_bulk() attempts to allocate N objects from a percpu
freelist of length M, and N > M > 0, it will first remove the M elements
from the percpu freelist, then call ___slab_alloc() to allocate the next
element and repopulate the percpu freelist. ___slab_alloc() can re-enable
IRQs via allocate_slab(), so the TID must be bumped before ___slab_alloc()
to properly commit the freelist head change.

Fix it by unconditionally bumping c->tid when entering the slowpath.

Cc: stable@vger.kernel.org
Fixes: ebe909e0fdb3 ("slub: improve bulk alloc strategy")
Signed-off-by: Jann Horn <jannh@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
{
	struct location *l;
	int order;

	order = get_order(sizeof(struct location) * max);

	l = (void *)__get_free_pages(flags, order);
	if (!l)
		return 0;

	if (t->count) {
		memcpy(l, t->loc, sizeof(struct location) * t->count);
		free_loc_track(t);
	}
	t->max = max;
	t->loc = l;
	return 1;
}


static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%lu\n", s->min_partial);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t poison_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
}


slab_flags_t kmem_cache_flags(unsigned int object_size,
	slab_flags_t flags, const char *name,
	void (*ctor)(void *))
{
	return flags;
}

#define __OBJECT_POISON		((slab_flags_t __force)0x80000000U)

#define __CMPXCHG_DOUBLE	((slab_flags_t __force)0x40000000U)

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\

static const struct attribute_group slab_attr_group = {
	.attrs = slab_attrs,
};

static const struct sysfs_ops slab_sysfs_ops = {
	.show = slab_attr_show,
	.store = slab_attr_store,
};

static const struct kset_uevent_ops slab_uevent_ops = {
	.filter = uevent_filter,
};

struct saved_alias {
	struct kmem_cache *s;
	const char *name;
	struct saved_alias *next;
}


void object_err(struct kmem_cache *s, struct page *page,
			u8 *object, char *reason)
{
	slab_bug(s, "%s", reason);
	print_trailer(s, page, object);
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static char *slub_debug_slabs;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static void process_slab(struct loc_track *t, struct kmem_cache *s,
		struct page *page, enum track_item alloc)
{
	void *addr = page_address(page);
	void *p;
	unsigned long *map;

	map = get_map(s, page);
	for_each_object(p, s, addr, page->objects)
		if (!test_bit(slab_index(p, s, addr), map))
			add_location(t, s, get_track(s, p, alloc));
	put_map(map);
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

enum track_item { TRACK_ALLOC, TRACK_FREE }

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", slub_cpu_partial(s));
}


static void init_object(struct kmem_cache *s, void *object, u8 val)
{
	u8 *p = object;

	if (s->flags & SLAB_RED_ZONE)
		memset(p - s->red_left_pad, val, s->red_left_pad);

	if (s->flags & __OBJECT_POISON) {
		memset(p, POISON_FREE, s->object_size - 1);
		p[s->object_size - 1] = POISON_END;
	}

	if (s->flags & SLAB_RED_ZONE)
		memset(p + s->object_size, val, s->inuse - s->object_size);
}

#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)

#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)

#define __OBJECT_POISON		((slab_flags_t __force)0x80000000U)

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
			int node, struct kmem_cache_cpu **pc)
{
	void *freelist;
	struct kmem_cache_cpu *c = *pc;
	struct page *page;

	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));

	freelist = get_partial(s, flags, node, c);

	if (freelist)
		return freelist;

	page = new_slab(s, flags, node);
	if (page) {
		c = raw_cpu_ptr(s->cpu_slab);
		if (c->page)
			flush_slab(s, c);

		/*
		 * No other reference to the page yet so we can
		 * muck around with it freely without cmpxchg
		 */
		freelist = page->freelist;
		page->freelist = NULL;

		stat(s, ALLOC_SLAB);
		c->page = page;
		*pc = c;
	}

	return freelist;
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static inline unsigned int slab_order(unsigned int size,
		unsigned int min_objects, unsigned int max_order,
		unsigned int fract_leftover)
{
	unsigned int min_order = slub_min_order;
	unsigned int order;

	if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
		return get_order(size * MAX_OBJS_PER_PAGE) - 1;

	for (order = max(min_order, (unsigned int)get_order(min_objects * size));
			order <= max_order; order++) {

		unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
		unsigned int rem;

		rem = slab_size % size;

		if (rem <= slab_size / fract_leftover)
			break;
	}

	return order;
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static ssize_t free_calls_show(struct kmem_cache *s, char *buf)
{
	if (!(s->flags & SLAB_STORE_USER))
		return -ENOSYS;
	return list_locations(s, buf, TRACK_FREE);
}


static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10);
}


static void sysfs_slab_remove_workfn(struct work_struct *work)
{
	struct kmem_cache *s =
		container_of(work, struct kmem_cache, kobj_remove_work);

	if (!s->kobj.state_in_sysfs)
		/*
		 * For a memcg cache, this may be called during
		 * deactivation and again on shutdown.  Remove only once.
		 * A cache is never shut down before deactivation is
		 * complete, so no need to worry about synchronization.
		 */
		goto out;

#ifdef CONFIG_MEMCG
	kset_unregister(s->memcg_kset);
#endif
	kobject_uevent(&s->kobj, KOBJ_REMOVE);
out:
	kobject_put(&s->kobj);


static __always_inline void kfree_hook(void *x)
{
	kmemleak_free(x);
	kasan_kfree_large(x, _RET_IP_);
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static inline int init_cache_random_seq(struct kmem_cache *s)
{
	return 0;
}


void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);

	trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
				s->size, gfpflags);

	return ret;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static int slab_memory_callback(struct notifier_block *self,
				unsigned long action, void *arg)
{
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		ret = slab_mem_going_online_callback(arg);
		break;
	case MEM_GOING_OFFLINE:
		ret = slab_mem_going_offline_callback(arg);
		break;
	case MEM_OFFLINE:
	case MEM_CANCEL_ONLINE:
		slab_mem_offline_callback(arg);
		break;
	case MEM_ONLINE:
	case MEM_CANCEL_OFFLINE:
		break;
	}
	if (ret)
		ret = notifier_from_errno(ret);
	else
		ret = NOTIFY_OK;
	return ret;
}


static noinline int free_debug_processing(
	struct kmem_cache *s, struct page *page,
	void *head, void *tail, int bulk_cnt,
	unsigned long addr)
{
	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
	void *object = head;
	int cnt = 0;
	unsigned long uninitialized_var(flags);
	int ret = 0;

	spin_lock_irqsave(&n->list_lock, flags);
	slab_lock(page);

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!check_slab(s, page))
			goto out;
	}

next_object:
	cnt++;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		if (!free_consistency_checks(s, page, object, addr))
			goto out;
	}

	if (s->flags & SLAB_STORE_USER)
		set_track(s, object, TRACK_FREE, addr);
	trace(s, page, object, 0);
	/* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
	init_object(s, object, SLUB_RED_INACTIVE);

	/* Reached end of constructed freelist yet? */
	if (object != tail) {
		object = get_freepointer(s, object);
		goto next_object;
	}
	ret = 1;

out:
	if (cnt != bulk_cnt)
		slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
			 bulk_cnt, cnt);

	slab_unlock(page);
	spin_unlock_irqrestore(&n->list_lock, flags);
	if (!ret)
		slab_fix(s, "Object at 0x%p not freed", object);
	return ret;
}

#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)

#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)

#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

enum track_item { TRACK_ALLOC, TRACK_FREE }

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static int uevent_filter(struct kset *kset, struct kobject *kobj)
{
	struct kobj_type *ktype = get_ktype(kobj);

	if (ktype == &slab_ktype)
		return 1;
	return 0;


void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
{
	if (WARN_ON(!size))
		return;

	do {
		struct detached_freelist df;

		size = build_detached_freelist(s, size, p, &df);
		if (!df.page)
			continue;

		slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
	} while (likely(size));
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
				 size_t length)
{
	unsigned long min;
	int err;

	err = kstrtoul(buf, 10, &min);
	if (err)
		return err;

	set_min_partial(s, min);
	return length;
}


static inline unsigned int slab_index(void *p, struct kmem_cache *s, void *addr)
{
	return (kasan_reset_tag(p) - addr) / s->size;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static struct track *get_track(struct kmem_cache *s, void *object,
	enum track_item alloc)
{
	struct track *p;

	if (s->offset)
		p = object + s->offset + sizeof(void *);
	else
		p = object + s->inuse;

	return p + alloc;
}


static inline unsigned int oo_order(struct kmem_cache_order_objects x)
{
	return x.x >> OO_SHIFT;
}

#define OO_SHIFT	16

#define OO_MASK		((1 << OO_SHIFT) - 1)

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static inline int alloc_debug_processing(struct kmem_cache *s,
	struct page *page, void *object, unsigned long addr) { return 0; }

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
}


static ssize_t reclaim_account_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	s->flags &= ~SLAB_RECLAIM_ACCOUNT;
	if (buf[0] == '1')
		s->flags |= SLAB_RECLAIM_ACCOUNT;
	return length;
}


static int slab_pad_check(struct kmem_cache *s, struct page *page)
{
	u8 *start;
	u8 *fault;
	u8 *end;
	u8 *pad;
	int length;
	int remainder;

	if (!(s->flags & SLAB_POISON))
		return 1;

	start = page_address(page);
	length = page_size(page);
	end = start + length;
	remainder = length % s->size;
	if (!remainder)
		return 1;

	pad = end - remainder;
	metadata_access_enable();
	fault = memchr_inv(pad, POISON_INUSE, remainder);
	metadata_access_disable();
	if (!fault)
		return 1;
	while (end > fault && end[-1] == POISON_INUSE)
		end--;

	slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
			fault, end - 1, fault - start);
	print_section(KERN_ERR, "Padding ", pad, remainder);

	restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
	return 0;
}

#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)

#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));

	/*
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
	 */
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);

	return 1;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


void kmem_cache_free(struct kmem_cache *s, void *x)
{
	s = cache_from_obj(s, x);
	if (!s)
		return;
	slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
	trace_kmem_cache_free(_RET_IP_, x);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static long validate_slab_cache(struct kmem_cache *s)
{
	int node;
	unsigned long count = 0;
	struct kmem_cache_node *n;

	flush_all(s);
	for_each_kmem_cache_node(s, node, n)
		count += validate_slab_node(s, n);

	return count;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t align_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->align);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t show_slab_objects(struct kmem_cache *s,
			    char *buf, unsigned long flags)
{
	unsigned long total = 0;
	int node;
	int x;
	unsigned long *nodes;

	nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
	if (!nodes)
		return -ENOMEM;

	if (flags & SO_CPU) {
		int cpu;

		for_each_possible_cpu(cpu) {
			struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
							       cpu);
			int node;
			struct page *page;

			page = READ_ONCE(c->page);
			if (!page)
				continue;

			node = page_to_nid(page);
			if (flags & SO_TOTAL)
				x = page->objects;
			else if (flags & SO_OBJECTS)
				x = page->inuse;
			else
				x = 1;

			total += x;
			nodes[node] += x;

			page = slub_percpu_partial_read_once(c);
			if (page) {
				node = page_to_nid(page);
				if (flags & SO_TOTAL)
					WARN_ON_ONCE(1);
				else if (flags & SO_OBJECTS)
					WARN_ON_ONCE(1);
				else
					x = page->pages;
				total += x;
				nodes[node] += x;
			}
		}
	}

	/*
	 * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
	 * already held which will conflict with an existing lock order:
	 *
	 * mem_hotplug_lock->slab_mutex->kernfs_mutex
	 *
	 * We don't really need mem_hotplug_lock (to hold off
	 * slab_mem_going_offline_callback) here because slab's memory hot
	 * unplug code doesn't destroy the kmem_cache->node[] data.
	 */

#ifdef CONFIG_SLUB_DEBUG
	if (flags & SO_ALL) {
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {

			if (flags & SO_TOTAL)
				x = atomic_long_read(&n->total_objects);
			else if (flags & SO_OBJECTS)
				x = atomic_long_read(&n->total_objects) -
					count_partial(n, count_free);
			else
				x = atomic_long_read(&n->nr_slabs);
			total += x;
			nodes[node] += x;
		}

	} else
#endif
	if (flags & SO_PARTIAL) {
		struct kmem_cache_node *n;

		for_each_kmem_cache_node(s, node, n) {
			if (flags & SO_TOTAL)
				x = count_partial(n, count_total);
			else if (flags & SO_OBJECTS)
				x = count_partial(n, count_inuse);
			else
				x = n->nr_partial;
			total += x;
			nodes[node] += x;
		}
	}
	x = sprintf(buf, "%lu", total);
#ifdef CONFIG_NUMA
	for (node = 0; node < nr_node_ids; node++)
		if (nodes[node])
			x += sprintf(buf + x, " N%d=%lu",
					node, nodes[node]);
#endif
	kfree(nodes);
	return x + sprintf(buf + x, "\n");
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

#define SO_ALL		(1 << SL_ALL)

#define SO_PARTIAL	(1 << SL_PARTIAL)

#define SO_CPU		(1 << SL_CPU)

#define SO_OBJECTS	(1 << SL_OBJECTS)

#define SO_TOTAL	(1 << SL_TOTAL)

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)


static void trace(struct kmem_cache *s, struct page *page, void *object,
								int alloc)
{
	if (s->flags & SLAB_TRACE) {
		pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
			s->name,
			alloc ? "alloc" : "free",
			object, page->inuse,
			page->freelist);

		if (!alloc)
			print_section(KERN_INFO, "Object ", (void *)object,
					s->object_size);

		dump_stack();
	}
}

#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s)
{
	/*
	 * Called with all the locks held after a sched RCU grace period.
	 * Even if @s becomes empty after shrinking, we can't know that @s
	 * doesn't have allocations already in-flight and thus can't
	 * destroy @s until the associated memcg is released.
	 *
	 * However, let's remove the sysfs files for empty caches here.
	 * Each cache has a lot of interface files which aren't
	 * particularly useful for empty draining caches; otherwise, we can
	 * easily end up with millions of unnecessary sysfs files on
	 * systems which have a lot of memory and transient cgroups.
	 */
	if (!__kmem_cache_shrink(s))
		sysfs_slab_remove(s);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->size);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static void set_min_partial(struct kmem_cache *s, unsigned long min)
{
	if (min < MIN_PARTIAL)
		min = MIN_PARTIAL;
	else if (min > MAX_PARTIAL)
		min = MAX_PARTIAL;
	s->min_partial = min;
}

#define MIN_PARTIAL 5

#define MAX_PARTIAL 10

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
		struct kmem_cache_cpu *c)
{
	void *object;
	int searchnode = node;

	if (node == NUMA_NO_NODE)
		searchnode = numa_mem_id();
	else if (!node_present_pages(node))
		searchnode = node_to_mem_node(node);

	object = get_partial_node(s, get_node(s, searchnode), c, flags);
	if (object || node != NUMA_NO_NODE)
		return object;

	return get_any_partial(s, flags, c);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static int count_free(struct page *page)
{
	return page->objects - page->inuse;
}


static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *p;
	unsigned long flags;

	local_irq_save(flags);
#ifdef CONFIG_PREEMPTION
	/*
	 * We may have been preempted and rescheduled on a different
	 * cpu before disabling interrupts. Need to reload cpu area
	 * pointer.
	 */
	c = this_cpu_ptr(s->cpu_slab);
#endif

	p = ___slab_alloc(s, gfpflags, node, addr, c);
	local_irq_restore(flags);
	return p;
}


static inline void sysfs_slab_remove(struct kmem_cache *s) { }


static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
							{ return 0; }


static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
{
	struct page *page;
	struct kmem_cache_order_objects oo = s->oo;
	gfp_t alloc_gfp;
	void *start, *p, *next;
	int idx;
	bool shuffle;

	flags &= gfp_allowed_mask;

	if (gfpflags_allow_blocking(flags))
		local_irq_enable();

	flags |= s->allocflags;

	/*
	 * Let the initial higher-order allocation fail under memory pressure
	 * so we fall-back to the minimum order allocation.
	 */
	alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
	if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
		alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);

	page = alloc_slab_page(s, alloc_gfp, node, oo);
	if (unlikely(!page)) {
		oo = s->min;
		alloc_gfp = flags;
		/*
		 * Allocation may have failed due to fragmentation.
		 * Try a lower order alloc if possible
		 */
		page = alloc_slab_page(s, alloc_gfp, node, oo);
		if (unlikely(!page))
			goto out;
		stat(s, ORDER_FALLBACK);
	}

	page->objects = oo_objects(oo);

	page->slab_cache = s;
	__SetPageSlab(page);
	if (page_is_pfmemalloc(page))
		SetPageSlabPfmemalloc(page);

	kasan_poison_slab(page);

	start = page_address(page);

	setup_page_debug(s, page, start);

	shuffle = shuffle_freelist(s, page);

	if (!shuffle) {
		start = fixup_red_left(s, start);
		start = setup_object(s, page, start);
		page->freelist = start;
		for (idx = 0, p = start; idx < page->objects - 1; idx++) {
			next = p + s->size;
			next = setup_object(s, page, next);
			set_freepointer(s, p, next);
			p = next;
		}
		set_freepointer(s, p, NULL);
	}

	page->inuse = page->objects;
	page->frozen = 1;

out:
	if (gfpflags_allow_blocking(flags))
		local_irq_disable();
	if (!page)
		return NULL;

	inc_slabs_node(s, page_to_nid(page), page->objects);

	return page;
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline unsigned int order_objects(unsigned int order, unsigned int size)
{
	return ((unsigned int)PAGE_SIZE << order) / size;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static inline void *freelist_dereference(const struct kmem_cache *s,
					 void *ptr_addr)
{
	return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
			    (unsigned long)ptr_addr);
}


static void __free_slab(struct kmem_cache *s, struct page *page)
{
	int order = compound_order(page);
	int pages = 1 << order;

	if (s->flags & SLAB_CONSISTENCY_CHECKS) {
		void *p;

		slab_pad_check(s, page);
		for_each_object(p, s, page_address(page),
						page->objects)
			check_object(s, page, p, SLUB_RED_INACTIVE);
	}

	__ClearPageSlabPfmemalloc(page);
	__ClearPageSlab(page);

	page->mapping = NULL;
	if (current->reclaim_state)
		current->reclaim_state->reclaimed_slab += pages;
	uncharge_slab_page(page, order, s);
	__free_pages(page, order);
}

#define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
				SLAB_POISON | SLAB_STORE_USER)

#define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
				SLAB_TRACE)

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)