MagickExport ChannelStatistics *GetImageChannelStatistics(const Image *image,
  ExceptionInfo *exception)
{
  ChannelStatistics
    *channel_statistics;

  double
    area,
    standard_deviation;

  MagickPixelPacket
    number_bins,
    *histogram;

  QuantumAny
    range;

  ssize_t
    i;

  size_t
    channels,
    depth,
    length;

  ssize_t
    y;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  length=CompositeChannels+1UL;
  channel_statistics=(ChannelStatistics *) AcquireQuantumMemory(length,
    sizeof(*channel_statistics));
  histogram=(MagickPixelPacket *) AcquireQuantumMemory(MaxMap+1U,
    sizeof(*histogram));
  if ((channel_statistics == (ChannelStatistics *) NULL) ||
      (histogram == (MagickPixelPacket *) NULL))
    {
      if (histogram != (MagickPixelPacket *) NULL)
        histogram=(MagickPixelPacket *) RelinquishMagickMemory(histogram);
      if (channel_statistics != (ChannelStatistics *) NULL)
        channel_statistics=(ChannelStatistics *) RelinquishMagickMemory(
          channel_statistics);
      return(channel_statistics);
    }
  (void) memset(channel_statistics,0,length*
    sizeof(*channel_statistics));
  for (i=0; i <= (ssize_t) CompositeChannels; i++)
  {
    channel_statistics[i].depth=1;
    channel_statistics[i].maxima=(-MagickMaximumValue);
    channel_statistics[i].minima=MagickMaximumValue;
  }
  (void) memset(histogram,0,(MaxMap+1U)*sizeof(*histogram));
  (void) memset(&number_bins,0,sizeof(number_bins));
  for (y=0; y < (ssize_t) image->rows; y++)
  {
    const IndexPacket
      *magick_restrict indexes;

    const PixelPacket
      *magick_restrict p;

    ssize_t
      x;

    /*
      Compute pixel statistics.
    */
    p=GetVirtualPixels(image,0,y,image->columns,1,exception);
    if (p == (const PixelPacket *) NULL)
      break;
    indexes=GetVirtualIndexQueue(image);
    for (x=0; x < (ssize_t) image->columns; )
    {
      if (channel_statistics[RedChannel].depth != MAGICKCORE_QUANTUM_DEPTH)
        {
          depth=channel_statistics[RedChannel].depth;
          range=GetQuantumRange(depth);
          if (IsPixelAtDepth(GetPixelRed(p),range) == MagickFalse)
            {
              channel_statistics[RedChannel].depth++;
              continue;
            }
        }
      if (channel_statistics[GreenChannel].depth != MAGICKCORE_QUANTUM_DEPTH)
        {
          depth=channel_statistics[GreenChannel].depth;
          range=GetQuantumRange(depth);
          if (IsPixelAtDepth(GetPixelGreen(p),range) == MagickFalse)
            {
              channel_statistics[GreenChannel].depth++;
              continue;
            }
        }
      if (channel_statistics[BlueChannel].depth != MAGICKCORE_QUANTUM_DEPTH)
        {
          depth=channel_statistics[BlueChannel].depth;
          range=GetQuantumRange(depth);
          if (IsPixelAtDepth(GetPixelBlue(p),range) == MagickFalse)
            {
              channel_statistics[BlueChannel].depth++;
              continue;
            }
        }
      if (image->matte != MagickFalse)
        {
          if (channel_statistics[OpacityChannel].depth != MAGICKCORE_QUANTUM_DEPTH)
            {
              depth=channel_statistics[OpacityChannel].depth;
              range=GetQuantumRange(depth);
              if (IsPixelAtDepth(GetPixelAlpha(p),range) == MagickFalse)
                {
                  channel_statistics[OpacityChannel].depth++;
                  continue;
                }
            }
          }
      if (image->colorspace == CMYKColorspace)
        {
          if (channel_statistics[BlackChannel].depth != MAGICKCORE_QUANTUM_DEPTH)
            {
              depth=channel_statistics[BlackChannel].depth;
              range=GetQuantumRange(depth);
              if (IsPixelAtDepth(GetPixelIndex(indexes+x),range) == MagickFalse)
                {
                  channel_statistics[BlackChannel].depth++;
                  continue;
                }
            }
        }
      if ((double) GetPixelRed(p) < channel_statistics[RedChannel].minima)
        channel_statistics[RedChannel].minima=(double) GetPixelRed(p);
      if ((double) GetPixelRed(p) > channel_statistics[RedChannel].maxima)
        channel_statistics[RedChannel].maxima=(double) GetPixelRed(p);
      channel_statistics[RedChannel].sum+=GetPixelRed(p);
      channel_statistics[RedChannel].sum_squared+=(double) GetPixelRed(p)*
        GetPixelRed(p);
      channel_statistics[RedChannel].sum_cubed+=(double)
        GetPixelRed(p)*GetPixelRed(p)*GetPixelRed(p);
      channel_statistics[RedChannel].sum_fourth_power+=(double)
        GetPixelRed(p)*GetPixelRed(p)*GetPixelRed(p)*GetPixelRed(p);
      if ((double) GetPixelGreen(p) < channel_statistics[GreenChannel].minima)
        channel_statistics[GreenChannel].minima=(double) GetPixelGreen(p);
      if ((double) GetPixelGreen(p) > channel_statistics[GreenChannel].maxima)
        channel_statistics[GreenChannel].maxima=(double) GetPixelGreen(p);
      channel_statistics[GreenChannel].sum+=GetPixelGreen(p);
      channel_statistics[GreenChannel].sum_squared+=(double) GetPixelGreen(p)*
        GetPixelGreen(p);
      channel_statistics[GreenChannel].sum_cubed+=(double) GetPixelGreen(p)*
        GetPixelGreen(p)*GetPixelGreen(p);
      channel_statistics[GreenChannel].sum_fourth_power+=(double)
        GetPixelGreen(p)*GetPixelGreen(p)*GetPixelGreen(p)*GetPixelGreen(p);
      if ((double) GetPixelBlue(p) < channel_statistics[BlueChannel].minima)
        channel_statistics[BlueChannel].minima=(double) GetPixelBlue(p);
      if ((double) GetPixelBlue(p) > channel_statistics[BlueChannel].maxima)
        channel_statistics[BlueChannel].maxima=(double) GetPixelBlue(p);
      channel_statistics[BlueChannel].sum+=GetPixelBlue(p);
      channel_statistics[BlueChannel].sum_squared+=(double) GetPixelBlue(p)*
        GetPixelBlue(p);
      channel_statistics[BlueChannel].sum_cubed+=(double) GetPixelBlue(p)*
        GetPixelBlue(p)*GetPixelBlue(p);
      channel_statistics[BlueChannel].sum_fourth_power+=(double)
        GetPixelBlue(p)*GetPixelBlue(p)*GetPixelBlue(p)*GetPixelBlue(p);
      histogram[ScaleQuantumToMap(GetPixelRed(p))].red++;
      histogram[ScaleQuantumToMap(GetPixelGreen(p))].green++;
      histogram[ScaleQuantumToMap(GetPixelBlue(p))].blue++;
      if (image->matte != MagickFalse)
        {
          if ((double) GetPixelAlpha(p) < channel_statistics[OpacityChannel].minima)
            channel_statistics[OpacityChannel].minima=(double) GetPixelAlpha(p);
          if ((double) GetPixelAlpha(p) > channel_statistics[OpacityChannel].maxima)
            channel_statistics[OpacityChannel].maxima=(double) GetPixelAlpha(p);
          channel_statistics[OpacityChannel].sum+=GetPixelAlpha(p);
          channel_statistics[OpacityChannel].sum_squared+=(double)
            GetPixelAlpha(p)*GetPixelAlpha(p);
          channel_statistics[OpacityChannel].sum_cubed+=(double)
            GetPixelAlpha(p)*GetPixelAlpha(p)*GetPixelAlpha(p);
          channel_statistics[OpacityChannel].sum_fourth_power+=(double)
            GetPixelAlpha(p)*GetPixelAlpha(p)*GetPixelAlpha(p)*GetPixelAlpha(p);
          histogram[ScaleQuantumToMap(GetPixelAlpha(p))].opacity++;
        }
      if (image->colorspace == CMYKColorspace)
        {
          if ((double) GetPixelIndex(indexes+x) < channel_statistics[BlackChannel].minima)
            channel_statistics[BlackChannel].minima=(double)
              GetPixelIndex(indexes+x);
          if ((double) GetPixelIndex(indexes+x) > channel_statistics[BlackChannel].maxima)
            channel_statistics[BlackChannel].maxima=(double)
              GetPixelIndex(indexes+x);
          channel_statistics[BlackChannel].sum+=GetPixelIndex(indexes+x);
          channel_statistics[BlackChannel].sum_squared+=(double)
            GetPixelIndex(indexes+x)*GetPixelIndex(indexes+x);
          channel_statistics[BlackChannel].sum_cubed+=(double)
            GetPixelIndex(indexes+x)*GetPixelIndex(indexes+x)*
            GetPixelIndex(indexes+x);
          channel_statistics[BlackChannel].sum_fourth_power+=(double)
            GetPixelIndex(indexes+x)*GetPixelIndex(indexes+x)*
            GetPixelIndex(indexes+x)*GetPixelIndex(indexes+x);
          histogram[ScaleQuantumToMap(GetPixelIndex(indexes+x))].index++;
        }
      x++;
      p++;
    }
  }
  for (i=0; i < (ssize_t) CompositeChannels; i++)
  {
    double
      area,
      mean,
      standard_deviation;

    /*
      Normalize pixel statistics.
    */
    area=PerceptibleReciprocal((double) image->columns*image->rows);
    mean=channel_statistics[i].sum*area;
    channel_statistics[i].sum=mean;
    channel_statistics[i].sum_squared*=area;
    channel_statistics[i].sum_cubed*=area;
    channel_statistics[i].sum_fourth_power*=area;
    channel_statistics[i].mean=mean;
    channel_statistics[i].variance=channel_statistics[i].sum_squared;
    standard_deviation=sqrt(channel_statistics[i].variance-(mean*mean));
    area=PerceptibleReciprocal((double) image->columns*image->rows-1.0)*
      ((double) image->columns*image->rows);
    standard_deviation=sqrt(area*standard_deviation*standard_deviation);
    channel_statistics[i].standard_deviation=standard_deviation;
  }
  for (i=0; i < (ssize_t) (MaxMap+1U); i++)
  {
    if (histogram[i].red > 0.0)
      number_bins.red++;
    if (histogram[i].green > 0.0)
      number_bins.green++;
    if (histogram[i].blue > 0.0)
      number_bins.blue++;
    if ((image->matte != MagickFalse) && (histogram[i].opacity > 0.0))
      number_bins.opacity++;
    if ((image->colorspace == CMYKColorspace) && (histogram[i].index > 0.0))
      number_bins.index++;
  }
  area=PerceptibleReciprocal((double) image->columns*image->rows);
  for (i=0; i < (ssize_t) (MaxMap+1U); i++)
  {
    /*
      Compute pixel entropy.
    */
    histogram[i].red*=area;
    channel_statistics[RedChannel].entropy+=-histogram[i].red*
      MagickLog10(histogram[i].red)*
      PerceptibleReciprocal(MagickLog10((double) number_bins.red));
    histogram[i].green*=area;
    channel_statistics[GreenChannel].entropy+=-histogram[i].green*
      MagickLog10(histogram[i].green)*
      PerceptibleReciprocal(MagickLog10((double) number_bins.green));
    histogram[i].blue*=area;
    channel_statistics[BlueChannel].entropy+=-histogram[i].blue*
      MagickLog10(histogram[i].blue)*
      PerceptibleReciprocal(MagickLog10((double) number_bins.blue));
    if (image->matte != MagickFalse)
      {
        histogram[i].opacity*=area;
        channel_statistics[OpacityChannel].entropy+=-histogram[i].opacity*
          MagickLog10(histogram[i].opacity)*
          PerceptibleReciprocal(MagickLog10((double) number_bins.opacity));
      }
    if (image->colorspace == CMYKColorspace)
      {
        histogram[i].index*=area;
        channel_statistics[IndexChannel].entropy+=-histogram[i].index*
          MagickLog10(histogram[i].index)*
          PerceptibleReciprocal(MagickLog10((double) number_bins.index));
      }
  }
  /*
    Compute overall statistics.
  */
  for (i=0; i < (ssize_t) CompositeChannels; i++)
  {
    channel_statistics[CompositeChannels].depth=(size_t) EvaluateMax((double)
      channel_statistics[CompositeChannels].depth,(double)
      channel_statistics[i].depth);
    channel_statistics[CompositeChannels].minima=MagickMin(
      channel_statistics[CompositeChannels].minima,
      channel_statistics[i].minima);
    channel_statistics[CompositeChannels].maxima=EvaluateMax(
      channel_statistics[CompositeChannels].maxima,
      channel_statistics[i].maxima);
    channel_statistics[CompositeChannels].sum+=channel_statistics[i].sum;
    channel_statistics[CompositeChannels].sum_squared+=
      channel_statistics[i].sum_squared;
    channel_statistics[CompositeChannels].sum_cubed+=
      channel_statistics[i].sum_cubed;
    channel_statistics[CompositeChannels].sum_fourth_power+=
      channel_statistics[i].sum_fourth_power;
    channel_statistics[CompositeChannels].mean+=channel_statistics[i].mean;
    channel_statistics[CompositeChannels].variance+=
      channel_statistics[i].variance-channel_statistics[i].mean*
      channel_statistics[i].mean;
    standard_deviation=sqrt(channel_statistics[i].variance-
      (channel_statistics[i].mean*channel_statistics[i].mean));
    area=PerceptibleReciprocal((double) image->columns*image->rows-1.0)*
      ((double) image->columns*image->rows);
    standard_deviation=sqrt(area*standard_deviation*standard_deviation);
    channel_statistics[CompositeChannels].standard_deviation=standard_deviation;
    channel_statistics[CompositeChannels].entropy+=
      channel_statistics[i].entropy;
  }
  channels=3;
  if (image->matte != MagickFalse)
    channels++;
  if (image->colorspace == CMYKColorspace)
    channels++;
  channel_statistics[CompositeChannels].sum/=channels;
  channel_statistics[CompositeChannels].sum_squared/=channels;
  channel_statistics[CompositeChannels].sum_cubed/=channels;
  channel_statistics[CompositeChannels].sum_fourth_power/=channels;
  channel_statistics[CompositeChannels].mean/=channels;
  channel_statistics[CompositeChannels].kurtosis/=channels;
  channel_statistics[CompositeChannels].skewness/=channels;
  channel_statistics[CompositeChannels].entropy/=channels;
  i=CompositeChannels;
  area=PerceptibleReciprocal((double) channels*image->columns*image->rows);
  channel_statistics[i].variance=channel_statistics[i].sum_squared;
  channel_statistics[i].mean=channel_statistics[i].sum;
  standard_deviation=sqrt(channel_statistics[i].variance-
    (channel_statistics[i].mean*channel_statistics[i].mean));
  standard_deviation=sqrt(PerceptibleReciprocal((double) channels*
    image->columns*image->rows-1.0)*channels*image->columns*image->rows*
    standard_deviation*standard_deviation);
  channel_statistics[i].standard_deviation=standard_deviation;
  for (i=0; i <= (ssize_t) CompositeChannels; i++)
  {
    /*
      Compute kurtosis & skewness statistics.
    */
    standard_deviation=PerceptibleReciprocal(
      channel_statistics[i].standard_deviation);
    channel_statistics[i].skewness=(channel_statistics[i].sum_cubed-3.0*
      channel_statistics[i].mean*channel_statistics[i].sum_squared+2.0*
      channel_statistics[i].mean*channel_statistics[i].mean*
      channel_statistics[i].mean)*(standard_deviation*standard_deviation*
      standard_deviation);
    channel_statistics[i].kurtosis=(channel_statistics[i].sum_fourth_power-4.0*
      channel_statistics[i].mean*channel_statistics[i].sum_cubed+6.0*
      channel_statistics[i].mean*channel_statistics[i].mean*
      channel_statistics[i].sum_squared-3.0*channel_statistics[i].mean*
      channel_statistics[i].mean*1.0*channel_statistics[i].mean*
      channel_statistics[i].mean)*(standard_deviation*standard_deviation*
      standard_deviation*standard_deviation)-3.0;
  }
  channel_statistics[CompositeChannels].mean=0.0;
  channel_statistics[CompositeChannels].standard_deviation=0.0;
  for (i=0; i < (ssize_t) CompositeChannels; i++)
  {
    channel_statistics[CompositeChannels].mean+=
      channel_statistics[i].mean;
    channel_statistics[CompositeChannels].standard_deviation+=
      channel_statistics[i].standard_deviation;
  }
  channel_statistics[CompositeChannels].mean/=(double) channels;
  channel_statistics[CompositeChannels].standard_deviation/=(double) channels;
  histogram=(MagickPixelPacket *) RelinquishMagickMemory(histogram);
  if (y < (ssize_t) image->rows)
    channel_statistics=(ChannelStatistics *) RelinquishMagickMemory(
      channel_statistics);
  return(channel_statistics);
}


static PixelList *AcquirePixelList(const size_t width,const size_t height)
{
  PixelList
    *pixel_list;

  ssize_t
    i;

  pixel_list=(PixelList *) AcquireMagickMemory(sizeof(*pixel_list));
  if (pixel_list == (PixelList *) NULL)
    return(pixel_list);
  (void) memset((void *) pixel_list,0,sizeof(*pixel_list));
  pixel_list->length=width*height;
  for (i=0; i < ListChannels; i++)
  {
    pixel_list->lists[i].nodes=(ListNode *) AcquireAlignedMemory(65537UL,
      sizeof(*pixel_list->lists[i].nodes));
    if (pixel_list->lists[i].nodes == (ListNode *) NULL)
      return(DestroyPixelList(pixel_list));
    (void) memset(pixel_list->lists[i].nodes,0,65537UL*
      sizeof(*pixel_list->lists[i].nodes));
  }
  pixel_list->signature=MagickCoreSignature;
  return(pixel_list);
}

#define ListChannels  5

typedef struct _ListNode
{
  size_t
    next[9],
    count,
    signature;
} ListNode;

typedef struct _SkipList
{
  ssize_t
    level;

  ListNode
    *nodes;
} SkipList;

typedef struct _PixelList
{
  size_t
    length,
    seed,
    signature;

  SkipList
    lists[ListChannels];
} PixelList;


static void GetRootMeanSquarePixelList(PixelList *pixel_list,
  MagickPixelPacket *pixel)
{
  MagickRealType
    sum;

  SkipList
    *list;

  ssize_t
    channel;

  size_t
    color;

  ssize_t
    count;

  unsigned short
    channels[ListChannels];

  /*
    Find the root mean square value for each of the color.
  */
  for (channel=0; channel < 5; channel++)
  {
    list=pixel_list->lists+channel;
    color=65536L;
    count=0;
    sum=0.0;
    do
    {
      color=list->nodes[color].next[0];
      sum+=(MagickRealType) (list->nodes[color].count*color*color);
      count+=list->nodes[color].count;
    } while (count < (ssize_t) pixel_list->length);
    sum/=pixel_list->length;
    channels[channel]=(unsigned short) sqrt(sum);
  }
  pixel->red=(MagickRealType) ScaleShortToQuantum(channels[0]);
  pixel->green=(MagickRealType) ScaleShortToQuantum(channels[1]);
  pixel->blue=(MagickRealType) ScaleShortToQuantum(channels[2]);
  pixel->opacity=(MagickRealType) ScaleShortToQuantum(channels[3]);
  pixel->index=(MagickRealType) ScaleShortToQuantum(channels[4]);
}


MagickExport Image *StatisticImageChannel(const Image *image,
  const ChannelType channel,const StatisticType type,const size_t width,
  const size_t height,ExceptionInfo *exception)
{
#define StatisticImageTag  "Statistic/Image"

  CacheView
    *image_view,
    *statistic_view;

  Image
    *statistic_image;

  MagickBooleanType
    status;

  MagickOffsetType
    progress;

  PixelList
    **magick_restrict pixel_list;

  size_t
    neighbor_height,
    neighbor_width;

  ssize_t
    y;

  /*
    Initialize statistics image attributes.
  */
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  statistic_image=CloneImage(image,0,0,MagickTrue,exception);
  if (statistic_image == (Image *) NULL)
    return((Image *) NULL);
  if (SetImageStorageClass(statistic_image,DirectClass) == MagickFalse)
    {
      InheritException(exception,&statistic_image->exception);
      statistic_image=DestroyImage(statistic_image);
      return((Image *) NULL);
    }
  neighbor_width=width == 0 ? GetOptimalKernelWidth2D((double) width,0.5) :
    width;
  neighbor_height=height == 0 ? GetOptimalKernelWidth2D((double) height,0.5) :
    height;
  pixel_list=AcquirePixelListThreadSet(neighbor_width,neighbor_height);
  if (pixel_list == (PixelList **) NULL)
    {
      statistic_image=DestroyImage(statistic_image);
      ThrowImageException(ResourceLimitError,"MemoryAllocationFailed");
    }
  /*
    Make each pixel the min / max / median / mode / etc. of the neighborhood.
  */
  status=MagickTrue;
  progress=0;
  image_view=AcquireVirtualCacheView(image,exception);
  statistic_view=AcquireAuthenticCacheView(statistic_image,exception);
#if defined(MAGICKCORE_OPENMP_SUPPORT)
  #pragma omp parallel for schedule(static) shared(progress,status) \
    magick_number_threads(image,statistic_image,statistic_image->rows,1)
#endif
  for (y=0; y < (ssize_t) statistic_image->rows; y++)
  {
    const int
      id = GetOpenMPThreadId();

    const IndexPacket
      *magick_restrict indexes;

    const PixelPacket
      *magick_restrict p;

    IndexPacket
      *magick_restrict statistic_indexes;

    PixelPacket
      *magick_restrict q;

    ssize_t
      x;

    if (status == MagickFalse)
      continue;
    p=GetCacheViewVirtualPixels(image_view,-((ssize_t) neighbor_width/2L),y-
      (ssize_t) (neighbor_height/2L),image->columns+neighbor_width,
      neighbor_height,exception);
    q=QueueCacheViewAuthenticPixels(statistic_view,0,y,statistic_image->columns,      1,exception);
    if ((p == (const PixelPacket *) NULL) || (q == (PixelPacket *) NULL))
      {
        status=MagickFalse;
        continue;
      }
    indexes=GetCacheViewVirtualIndexQueue(image_view);
    statistic_indexes=GetCacheViewAuthenticIndexQueue(statistic_view);
    for (x=0; x < (ssize_t) statistic_image->columns; x++)
    {
      MagickPixelPacket
        pixel;

      const IndexPacket
        *magick_restrict s;

      const PixelPacket
        *magick_restrict r;

      ssize_t
        u,
        v;

      r=p;
      s=indexes+x;
      ResetPixelList(pixel_list[id]);
      for (v=0; v < (ssize_t) neighbor_height; v++)
      {
        for (u=0; u < (ssize_t) neighbor_width; u++)
          InsertPixelList(image,r+u,s+u,pixel_list[id]);
        r+=image->columns+neighbor_width;
        s+=image->columns+neighbor_width;
      }
      GetMagickPixelPacket(image,&pixel);
      SetMagickPixelPacket(image,p+neighbor_width*neighbor_height/2,indexes+x+
        neighbor_width*neighbor_height/2,&pixel);
      switch (type)
      {
        case GradientStatistic:
        {
          MagickPixelPacket
            maximum,
            minimum;

          GetMinimumPixelList(pixel_list[id],&pixel);
          minimum=pixel;
          GetMaximumPixelList(pixel_list[id],&pixel);
          maximum=pixel;
          pixel.red=MagickAbsoluteValue(maximum.red-minimum.red);
          pixel.green=MagickAbsoluteValue(maximum.green-minimum.green);
          pixel.blue=MagickAbsoluteValue(maximum.blue-minimum.blue);
          pixel.opacity=MagickAbsoluteValue(maximum.opacity-minimum.opacity);
          if (image->colorspace == CMYKColorspace)
            pixel.index=MagickAbsoluteValue(maximum.index-minimum.index);
          break;
        }
        case MaximumStatistic:
        {
          GetMaximumPixelList(pixel_list[id],&pixel);
          break;
        }
        case MeanStatistic:
        {
          GetMeanPixelList(pixel_list[id],&pixel);
          break;
        }
        case MedianStatistic:
        default:
        {
          GetMedianPixelList(pixel_list[id],&pixel);
          break;
        }
        case MinimumStatistic:
        {
          GetMinimumPixelList(pixel_list[id],&pixel);
          break;
        }
        case ModeStatistic:
        {
          GetModePixelList(pixel_list[id],&pixel);
          break;
        }
        case NonpeakStatistic:
        {
          GetNonpeakPixelList(pixel_list[id],&pixel);
          break;
        }
        case RootMeanSquareStatistic:
        {
          GetRootMeanSquarePixelList(pixel_list[id],&pixel);
          break;
        }
        case StandardDeviationStatistic:
        {
          GetStandardDeviationPixelList(pixel_list[id],&pixel);
          break;
        }
      }
      if ((channel & RedChannel) != 0)
        SetPixelRed(q,ClampToQuantum(pixel.red));
      if ((channel & GreenChannel) != 0)
        SetPixelGreen(q,ClampToQuantum(pixel.green));
      if ((channel & BlueChannel) != 0)
        SetPixelBlue(q,ClampToQuantum(pixel.blue));
      if ((channel & OpacityChannel) != 0)
        SetPixelOpacity(q,ClampToQuantum(pixel.opacity));
      if (((channel & IndexChannel) != 0) &&
          (image->colorspace == CMYKColorspace))
        SetPixelIndex(statistic_indexes+x,ClampToQuantum(pixel.index));
      p++;
      q++;
    }
    if (SyncCacheViewAuthenticPixels(statistic_view,exception) == MagickFalse)
      status=MagickFalse;
    if (image->progress_monitor != (MagickProgressMonitor) NULL)
      {
        MagickBooleanType
          proceed;

        proceed=SetImageProgress(image,StatisticImageTag,progress++,
          image->rows);
        if (proceed == MagickFalse)
          status=MagickFalse;
      }
  }
  statistic_view=DestroyCacheView(statistic_view);
  image_view=DestroyCacheView(image_view);
  pixel_list=DestroyPixelListThreadSet(pixel_list);
  if (status == MagickFalse)
    statistic_image=DestroyImage(statistic_image);
  return(statistic_image);
}


MagickExport MagickBooleanType GetImageChannelMean(const Image *image,
  const ChannelType channel,double *mean,double *standard_deviation,
  ExceptionInfo *exception)
{
  ChannelStatistics
    *channel_statistics;

  size_t
    channels;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  channel_statistics=GetImageChannelStatistics(image,exception);
  if (channel_statistics == (ChannelStatistics *) NULL)
    return(MagickFalse);
  channels=0;
  channel_statistics[CompositeChannels].mean=0.0;
  channel_statistics[CompositeChannels].standard_deviation=0.0;
  if ((channel & RedChannel) != 0)
    {
      channel_statistics[CompositeChannels].mean+=
        channel_statistics[RedChannel].mean;
      channel_statistics[CompositeChannels].standard_deviation+=
        channel_statistics[RedChannel].standard_deviation;
      channels++;
    }
  if ((channel & GreenChannel) != 0)
    {
      channel_statistics[CompositeChannels].mean+=
        channel_statistics[GreenChannel].mean;
      channel_statistics[CompositeChannels].standard_deviation+=
        channel_statistics[GreenChannel].standard_deviation;
      channels++;
    }
  if ((channel & BlueChannel) != 0)
    {
      channel_statistics[CompositeChannels].mean+=
        channel_statistics[BlueChannel].mean;
      channel_statistics[CompositeChannels].standard_deviation+=
        channel_statistics[BlueChannel].standard_deviation;
      channels++;
    }
  if (((channel & OpacityChannel) != 0) && (image->matte != MagickFalse))
    {
      channel_statistics[CompositeChannels].mean+=
        (QuantumRange-channel_statistics[OpacityChannel].mean);
      channel_statistics[CompositeChannels].standard_deviation+=
        channel_statistics[OpacityChannel].standard_deviation;
      channels++;
    }
  if (((channel & IndexChannel) != 0) && (image->colorspace == CMYKColorspace))
    {
      channel_statistics[CompositeChannels].mean+=
        channel_statistics[BlackChannel].mean;
      channel_statistics[CompositeChannels].standard_deviation+=
        channel_statistics[CompositeChannels].standard_deviation;
      channels++;
    }
  channel_statistics[CompositeChannels].mean/=channels;
  channel_statistics[CompositeChannels].standard_deviation/=channels;
  *mean=channel_statistics[CompositeChannels].mean;
  *standard_deviation=channel_statistics[CompositeChannels].standard_deviation;
  channel_statistics=(ChannelStatistics *) RelinquishMagickMemory(
    channel_statistics);
  return(MagickTrue);
}


MagickExport MagickBooleanType GetImageRange(const Image *image,
  double *minima,double *maxima,ExceptionInfo *exception)
{
  return(GetImageChannelRange(image,CompositeChannels,minima,maxima,exception));
}


static Image *AcquireImageCanvas(const Image *images,ExceptionInfo *exception)
{
  const Image
    *p,
    *q;

  size_t
    columns,
    number_channels,
    rows;

  q=images;
  columns=images->columns;
  rows=images->rows;
  number_channels=0;
  for (p=images; p != (Image *) NULL; p=p->next)
  {
    size_t
      channels;

    channels=3;
    if (p->matte != MagickFalse)
      channels+=1;
    if (p->colorspace == CMYKColorspace)
      channels+=1;
    if (channels > number_channels)
      {
        number_channels=channels;
        q=p;
      }
    if (p->columns > columns)
      columns=p->columns;
    if (p->rows > rows)
      rows=p->rows;
  }
  return(CloneImage(q,columns,rows,MagickTrue,exception));
}


void remove_pending_change_notify_requests_by_fid(files_struct *fsp,
						  NTSTATUS status)
{
	if (fsp->notify == NULL) {
		return;
	}

	while (fsp->notify->requests != NULL) {
		change_notify_reply(fsp->notify->requests->req,
				    status, 0, NULL,
				    fsp->notify->requests->reply_fn);
		change_notify_remove_request(fsp->conn->sconn,
					     fsp->notify->requests);
	}
}

s3: smbd: Ensure change notifies can't get set unless the directory handle is open for SEC_DIR_LIST.

Remove knownfail entry.

CVE-2020-14318

BUG: https://bugzilla.samba.org/show_bug.cgi?id=14434

Signed-off-by: Jeremy Allison <jra@samba.org>

struct notify_change_buf {
	/*
	 * Filters for reinitializing after notifyd has been restarted
	 */
	uint32_t filter;
	uint32_t subdir_filter;

	/*
	 * If no requests are pending, changes are queued here. Simple array,
	 * we only append.
	 */

	uint32_t max_buffer_size;

	/*
	 * num_changes == -1 means that we have got a catch-all change, when
	 * asked we just return NT_STATUS_OK without specific changes.
	 */
	int num_changes;
	struct notify_change_event *changes;

	/*
	 * If no changes are around requests are queued here. Using a linked
	 * list, because we have to append at the end and delete from the top.
	 */
	struct notify_change_request *requests;
}

struct notify_change_request {
	struct notify_change_request *prev, *next;
	struct files_struct *fsp;	/* backpointer for cancel by mid */
	struct smb_request *req;
	uint32_t filter;
	uint32_t max_param;
	void (*reply_fn)(struct smb_request *req,
			 NTSTATUS error_code,
			 uint8_t *buf, size_t len);
	struct notify_mid_map *mid_map;
	void *backend_data;
}

static void notify_fsp(files_struct *fsp, struct timespec when,
		       uint32_t action, const char *name);

struct notify_mid_map {
	struct notify_mid_map *prev, *next;
	struct notify_change_request *req;
	uint64_t mid;
}

struct notify_fsp_state {
	struct files_struct *notified_fsp;
	struct timespec when;
	const struct notify_event *e;
}


void notify_fname(connection_struct *conn, uint32_t action, uint32_t filter,
		  const char *path)
{
	struct notify_context *notify_ctx = conn->sconn->notify_ctx;

	if (path[0] == '.' && path[1] == '/') {
		path += 2;
	}

	notify_trigger(notify_ctx, action, filter, conn->connectpath, path);
}

struct notify_change_event {
	struct timespec when;
	uint32_t action;
	const char *name;
}

struct notify_change_buf {
	/*
	 * Filters for reinitializing after notifyd has been restarted
	 */
	uint32_t filter;
	uint32_t subdir_filter;

	/*
	 * If no requests are pending, changes are queued here. Simple array,
	 * we only append.
	 */

	uint32_t max_buffer_size;

	/*
	 * num_changes == -1 means that we have got a catch-all change, when
	 * asked we just return NT_STATUS_OK without specific changes.
	 */
	int num_changes;
	struct notify_change_event *changes;

	/*
	 * If no changes are around requests are queued here. Using a linked
	 * list, because we have to append at the end and delete from the top.
	 */
	struct notify_change_request *requests;
}

struct notify_change_request {
	struct notify_change_request *prev, *next;
	struct files_struct *fsp;	/* backpointer for cancel by mid */
	struct smb_request *req;
	uint32_t filter;
	uint32_t max_param;
	void (*reply_fn)(struct smb_request *req,
			 NTSTATUS error_code,
			 uint8_t *buf, size_t len);
	struct notify_mid_map *mid_map;
	void *backend_data;
}

static void notify_fsp(files_struct *fsp, struct timespec when,
		       uint32_t action, const char *name);

struct notify_mid_map {
	struct notify_mid_map *prev, *next;
	struct notify_change_request *req;
	uint64_t mid;
}

struct notify_fsp_state {
	struct files_struct *notified_fsp;
	struct timespec when;
	const struct notify_event *e;
}


bool remove_pending_change_notify_requests_by_mid(
	struct smbd_server_connection *sconn, uint64_t mid)
{
	struct notify_mid_map *map;

	for (map = sconn->smb1.notify_mid_maps; map; map = map->next) {
		if (map->mid == mid) {
			break;
		}
	}

	if (map == NULL) {
		return false;
	}

	smbd_notify_cancel_by_map(map);
	return true;
}


static bool notify_marshall_changes(int num_changes,
				uint32_t max_offset,
				struct notify_change_event *changes,
				DATA_BLOB *final_blob)
{
	int i;

	if (num_changes == -1) {
		return false;
	}

	/*
	 * Sort the notifies by timestamp when the event happened to avoid
	 * coalescing and thus dropping events.
	 */

	qsort(changes, num_changes,
	      sizeof(*changes), compare_notify_change_events);

	for (i=0; i<num_changes; i++) {
		enum ndr_err_code ndr_err;
		struct notify_change_event *c;
		struct FILE_NOTIFY_INFORMATION m;
		DATA_BLOB blob;
		uint16_t pad = 0;

		/* Coalesce any identical records. */
		while (i+1 < num_changes &&
			notify_change_record_identical(&changes[i],
						&changes[i+1])) {
			i++;
		}

		c = &changes[i];

		m.FileName1 = c->name;
		m.FileNameLength = strlen_m(c->name)*2;
		m.Action = c->action;

		m._pad = data_blob_null;

		/*
		 * Offset to next entry, only if there is one
		 */

		if (i == (num_changes-1)) {
			m.NextEntryOffset = 0;
		} else {
			if ((m.FileNameLength % 4) == 2) {
				m._pad = data_blob_const(&pad, 2);
			}
			m.NextEntryOffset =
				ndr_size_FILE_NOTIFY_INFORMATION(&m, 0);
		}

		ndr_err = ndr_push_struct_blob(&blob, talloc_tos(), &m,
			(ndr_push_flags_fn_t)ndr_push_FILE_NOTIFY_INFORMATION);
		if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
			return false;
		}

		if (DEBUGLEVEL >= 10) {
			NDR_PRINT_DEBUG(FILE_NOTIFY_INFORMATION, &m);
		}

		if (!data_blob_append(talloc_tos(), final_blob,
				      blob.data, blob.length)) {
			data_blob_free(&blob);
			return false;
		}

		data_blob_free(&blob);

		if (final_blob->length > max_offset) {
			/* Too much data for client. */
			DEBUG(10, ("Client only wanted %d bytes, trying to "
				   "marshall %d bytes\n", (int)max_offset,
				   (int)final_blob->length));
			return False;
		}
	}

	return True;
}


void smbd_notify_cancel_deleted(struct messaging_context *msg,
				void *private_data, uint32_t msg_type,
				struct server_id server_id, DATA_BLOB *data)
{
	struct smbd_server_connection *sconn = talloc_get_type_abort(
		private_data, struct smbd_server_connection);
	struct file_id *fid;
	enum ndr_err_code ndr_err;

	fid = talloc(talloc_tos(), struct file_id);
	if (fid == NULL) {
		DEBUG(1, ("talloc failed\n"));
		return;
	}

	ndr_err = ndr_pull_struct_blob_all(
		data, fid, fid, (ndr_pull_flags_fn_t)ndr_pull_file_id);
	if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
		DEBUG(10, ("%s: ndr_pull_file_id failed: %s\n", __func__,
			   ndr_errstr(ndr_err)));
		goto done;
	}

	files_forall(sconn, smbd_notify_cancel_deleted_fn, fid);

done:
	TALLOC_FREE(fid);
}


void smbd_notifyd_restarted(struct messaging_context *msg,
			    void *private_data, uint32_t msg_type,
			    struct server_id server_id, DATA_BLOB *data)
{
	struct smbd_server_connection *sconn = talloc_get_type_abort(
		private_data, struct smbd_server_connection);

	TALLOC_FREE(sconn->notify_ctx);

	sconn->notify_ctx = notify_init(sconn, sconn->msg_ctx,
					sconn, notify_callback);
	if (sconn->notify_ctx == NULL) {
		DBG_DEBUG("notify_init failed\n");
		return;
	}

	files_forall(sconn, smbd_notifyd_reregister, sconn->notify_ctx);
}


static int compare_notify_change_events(const void *p1, const void *p2)
{
	const struct notify_change_event *e1 = p1;
	const struct notify_change_event *e2 = p2;

	return timespec_compare(&e1->when, &e2->when);
}


static struct files_struct *smbd_notifyd_reregister(struct files_struct *fsp,
						    void *private_data)
{
	DBG_DEBUG("reregister %s\n", fsp->fsp_name->base_name);

	if ((fsp->conn->sconn->notify_ctx != NULL) &&
	    (fsp->notify != NULL) &&
	    ((fsp->notify->filter != 0) ||
	     (fsp->notify->subdir_filter != 0))) {
		size_t len = fsp_fullbasepath(fsp, NULL, 0);
		char fullpath[len+1];

		NTSTATUS status;

		fsp_fullbasepath(fsp, fullpath, sizeof(fullpath));
		if (len > 1 && fullpath[len-1] == '.' &&
		    fullpath[len-2] == '/') {
			fullpath[len-2] = '\0';
		}

		status = notify_add(fsp->conn->sconn->notify_ctx,
				    fullpath, fsp->notify->filter,
				    fsp->notify->subdir_filter, fsp);
		if (!NT_STATUS_IS_OK(status)) {
			DBG_DEBUG("notify_add failed: %s\n",
				  nt_errstr(status));
		}
	}
	return NULL;
}


void smbd_notify_cancel_by_smbreq(const struct smb_request *smbreq)
{
	struct smbd_server_connection *sconn = smbreq->sconn;
	struct notify_mid_map *map;

	for (map = sconn->smb1.notify_mid_maps; map; map = map->next) {
		if (map->req->req == smbreq) {
			break;
		}
	}

	if (map == NULL) {
		return;
	}

	smbd_notify_cancel_by_map(map);
}


NTSTATUS change_notify_add_request(struct smb_request *req,
				uint32_t max_param,
				uint32_t filter, bool recursive,
				struct files_struct *fsp,
				void (*reply_fn)(struct smb_request *req,
					NTSTATUS error_code,
					uint8_t *buf, size_t len))
{
	struct notify_change_request *request = NULL;
	struct notify_mid_map *map = NULL;
	struct smbd_server_connection *sconn = req->sconn;

	DEBUG(10, ("change_notify_add_request: Adding request for %s: "
		   "max_param = %d\n", fsp_str_dbg(fsp), (int)max_param));

	if (!(request = talloc(NULL, struct notify_change_request))
	    || !(map = talloc(request, struct notify_mid_map))) {
		TALLOC_FREE(request);
		return NT_STATUS_NO_MEMORY;
	}

	request->mid_map = map;
	map->req = request;

	request->req = talloc_move(request, &req);
	request->max_param = max_param;
	request->filter = filter;
	request->fsp = fsp;
	request->reply_fn = reply_fn;
	request->backend_data = NULL;

	DLIST_ADD_END(fsp->notify->requests, request);

	map->mid = request->req->mid;
	DLIST_ADD(sconn->smb1.notify_mid_maps, map);

	return NT_STATUS_OK;
}


static struct files_struct *smbd_notify_cancel_deleted_fn(
	struct files_struct *fsp, void *private_data)
{
	struct file_id *fid = talloc_get_type_abort(
		private_data, struct file_id);

	if (file_id_equal(&fsp->file_id, fid)) {
		remove_pending_change_notify_requests_by_fid(
			fsp, NT_STATUS_DELETE_PENDING);
	}
	return NULL;
}


static void notify_fsp(files_struct *fsp, struct timespec when,
		       uint32_t action, const char *name)
{
	struct notify_change_event *change, *changes;
	char *tmp;

	if (fsp->notify == NULL) {
		/*
		 * Nobody is waiting, don't queue
		 */
		return;
	}

	/*
	 * Someone has triggered a notify previously, queue the change for
	 * later.
	 */

	if ((fsp->notify->num_changes > 1000) || (name == NULL)) {
		/*
		 * The real number depends on the client buf, just provide a
		 * guard against a DoS here.  If name == NULL the CN backend is
		 * alerting us to a problem.  Possibly dropped events.  Clear
		 * queued changes and send the catch-all response to the client
		 * if a request is pending.
		 */
		TALLOC_FREE(fsp->notify->changes);
		fsp->notify->num_changes = -1;
		if (fsp->notify->requests != NULL) {
			change_notify_reply(fsp->notify->requests->req,
					    NT_STATUS_OK,
					    fsp->notify->requests->max_param,
					    fsp->notify,
					    fsp->notify->requests->reply_fn);
			change_notify_remove_request(fsp->conn->sconn,
						     fsp->notify->requests);
		}
		return;
	}

	/* If we've exceeded the server side queue or received a NULL name
	 * from the underlying CN implementation, don't queue up any more
	 * requests until we can send a catch-all response to the client */
	if (fsp->notify->num_changes == -1) {
		return;
	}

	if (!(changes = talloc_realloc(
		      fsp->notify, fsp->notify->changes,
		      struct notify_change_event,
		      fsp->notify->num_changes+1))) {
		DEBUG(0, ("talloc_realloc failed\n"));
		return;
	}

	fsp->notify->changes = changes;

	change = &(fsp->notify->changes[fsp->notify->num_changes]);

	if (!(tmp = talloc_strdup(changes, name))) {
		DEBUG(0, ("talloc_strdup failed\n"));
		return;
	}

	string_replace(tmp, '/', '\\');
	change->name = tmp;

	change->when = when;
	change->action = action;
	fsp->notify->num_changes += 1;

	if (fsp->notify->requests == NULL) {
		/*
		 * Nobody is waiting, so don't send anything. The ot
		 */
		return;
	}

	if (action == NOTIFY_ACTION_OLD_NAME) {
		/*
		 * We have to send the two rename events in one reply. So hold
		 * the first part back.
		 */
		return;
	}

	/*
	 * Someone is waiting for the change, trigger the reply immediately.
	 *
	 * TODO: do we have to walk the lists of requests pending?
	 */

	change_notify_reply(fsp->notify->requests->req,
			    NT_STATUS_OK,
			    fsp->notify->requests->max_param,
			    fsp->notify,
			    fsp->notify->requests->reply_fn);

	change_notify_remove_request(fsp->conn->sconn, fsp->notify->requests);
}


static struct files_struct *notify_fsp_cb(struct files_struct *fsp,
					  void *private_data)
{
	struct notify_fsp_state *state = private_data;

	if (fsp == state->notified_fsp) {
		DBG_DEBUG("notify_callback called for %s\n", fsp_str_dbg(fsp));
		notify_fsp(fsp, state->when, state->e->action, state->e->path);
		return fsp;
	}

	return NULL;
}


bool change_notify_fsp_has_changes(struct files_struct *fsp)
{
	if (fsp == NULL) {
		return false;
	}

	if (fsp->notify == NULL) {
		return false;
	}

	if (fsp->notify->num_changes == 0) {
		return false;
	}

	return true;
}


NTSTATUS change_notify_create(struct files_struct *fsp,
			      uint32_t max_buffer_size,
			      uint32_t filter,
			      bool recursive)
{
	size_t len = fsp_fullbasepath(fsp, NULL, 0);
	char fullpath[len+1];
	NTSTATUS status = NT_STATUS_NOT_IMPLEMENTED;

	/*
	 * Setting a changenotify needs READ/LIST access
	 * on the directory handle.
	 */
	if (!(fsp->access_mask & SEC_DIR_LIST)) {
		return NT_STATUS_ACCESS_DENIED;
	}

	if (fsp->notify != NULL) {
		DEBUG(1, ("change_notify_create: fsp->notify != NULL, "
			  "fname = %s\n", fsp->fsp_name->base_name));
		return NT_STATUS_INVALID_PARAMETER;
	}

	if (!(fsp->notify = talloc_zero(NULL, struct notify_change_buf))) {
		DEBUG(0, ("talloc failed\n"));
		return NT_STATUS_NO_MEMORY;
	}
	fsp->notify->filter = filter;
	fsp->notify->subdir_filter = recursive ? filter : 0;
	fsp->notify->max_buffer_size = max_buffer_size;

	fsp_fullbasepath(fsp, fullpath, sizeof(fullpath));

	/*
	 * Avoid /. at the end of the path name. notify can't deal with it.
	 */
	if (len > 1 && fullpath[len-1] == '.' && fullpath[len-2] == '/') {
		fullpath[len-2] = '\0';
	}

	if ((fsp->notify->filter != 0) ||
	    (fsp->notify->subdir_filter != 0)) {
		status = notify_add(fsp->conn->sconn->notify_ctx,
				    fullpath, fsp->notify->filter,
				    fsp->notify->subdir_filter, fsp);
	}

	return status;
}


struct sys_notify_context *sys_notify_context_create(TALLOC_CTX *mem_ctx,
						     struct tevent_context *ev)
{
	struct sys_notify_context *ctx;

	if (!(ctx = talloc(mem_ctx, struct sys_notify_context))) {
		DEBUG(0, ("talloc failed\n"));
		return NULL;
	}

	ctx->ev = ev;
	ctx->private_data = NULL;
	return ctx;
}


static void smbd_notify_cancel_by_map(struct notify_mid_map *map)
{
	struct smb_request *smbreq = map->req->req;
	struct smbd_server_connection *sconn = smbreq->sconn;
	struct smbd_smb2_request *smb2req = smbreq->smb2req;
	NTSTATUS notify_status = NT_STATUS_CANCELLED;

	if (smb2req != NULL) {
		NTSTATUS sstatus;

		if (smb2req->session == NULL) {
			sstatus = NT_STATUS_USER_SESSION_DELETED;
		} else {
			sstatus = smb2req->session->status;
		}

		if (NT_STATUS_EQUAL(sstatus, NT_STATUS_NETWORK_SESSION_EXPIRED)) {
			sstatus = NT_STATUS_OK;
		}

		if (!NT_STATUS_IS_OK(sstatus)) {
			notify_status = STATUS_NOTIFY_CLEANUP;
		} else if (smb2req->tcon == NULL) {
			notify_status = STATUS_NOTIFY_CLEANUP;
		} else if (!NT_STATUS_IS_OK(smb2req->tcon->status)) {
			notify_status = STATUS_NOTIFY_CLEANUP;
		}
	}

	change_notify_reply(smbreq, notify_status,
			    0, NULL, map->req->reply_fn);
	change_notify_remove_request(sconn, map->req);
}


static bool notify_change_record_identical(struct notify_change_event *c1,
					   struct notify_change_event *c2)
{
	/* Note this is deliberately case sensitive. */
	if (c1->action == c2->action &&
			strcmp(c1->name, c2->name) == 0) {
		return True;
	}
	return False;
}


void change_notify_reply(struct smb_request *req,
			 NTSTATUS error_code,
			 uint32_t max_param,
			 struct notify_change_buf *notify_buf,
			 void (*reply_fn)(struct smb_request *req,
					  NTSTATUS error_code,
					  uint8_t *buf, size_t len))
{
	DATA_BLOB blob = data_blob_null;

	if (!NT_STATUS_IS_OK(error_code)) {
		reply_fn(req, error_code, NULL, 0);
		return;
	}

	if (notify_buf == NULL) {
		reply_fn(req, NT_STATUS_OK, NULL, 0);
		return;
	}

	max_param = MIN(max_param, notify_buf->max_buffer_size);

	if (!notify_marshall_changes(notify_buf->num_changes, max_param,
					notify_buf->changes, &blob)) {
		/*
		 * We exceed what the client is willing to accept. Send
		 * nothing.
		 */
		data_blob_free(&blob);
	}

	reply_fn(req, NT_STATUS_OK, blob.data, blob.length);

	data_blob_free(&blob);

	TALLOC_FREE(notify_buf->changes);
	notify_buf->num_changes = 0;
}


static void change_notify_remove_request(struct smbd_server_connection *sconn,
					 struct notify_change_request *remove_req)
{
	files_struct *fsp;
	struct notify_change_request *req;

	/*
	 * Paranoia checks, the fsp referenced must must have the request in
	 * its list of pending requests
	 */

	fsp = remove_req->fsp;
	SMB_ASSERT(fsp->notify != NULL);

	for (req = fsp->notify->requests; req; req = req->next) {
		if (req == remove_req) {
			break;
		}
	}

	if (req == NULL) {
		smb_panic("notify_req not found in fsp's requests");
	}

	DLIST_REMOVE(fsp->notify->requests, req);
	DLIST_REMOVE(sconn->smb1.notify_mid_maps, req->mid_map);
	TALLOC_FREE(req);
}


char *notify_filter_string(TALLOC_CTX *mem_ctx, uint32_t filter)
{
	char *result = NULL;

	result = talloc_strdup(mem_ctx, "");

	if (filter & FILE_NOTIFY_CHANGE_FILE_NAME)
		result = talloc_asprintf_append(result, "FILE_NAME|");
	if (filter & FILE_NOTIFY_CHANGE_DIR_NAME)
		result = talloc_asprintf_append(result, "DIR_NAME|");
	if (filter & FILE_NOTIFY_CHANGE_ATTRIBUTES)
		result = talloc_asprintf_append(result, "ATTRIBUTES|");
	if (filter & FILE_NOTIFY_CHANGE_SIZE)
		result = talloc_asprintf_append(result, "SIZE|");
	if (filter & FILE_NOTIFY_CHANGE_LAST_WRITE)
		result = talloc_asprintf_append(result, "LAST_WRITE|");
	if (filter & FILE_NOTIFY_CHANGE_LAST_ACCESS)
		result = talloc_asprintf_append(result, "LAST_ACCESS|");
	if (filter & FILE_NOTIFY_CHANGE_CREATION)
		result = talloc_asprintf_append(result, "CREATION|");
	if (filter & FILE_NOTIFY_CHANGE_EA)
		result = talloc_asprintf_append(result, "EA|");
	if (filter & FILE_NOTIFY_CHANGE_SECURITY)
		result = talloc_asprintf_append(result, "SECURITY|");
	if (filter & FILE_NOTIFY_CHANGE_STREAM_NAME)
		result = talloc_asprintf_append(result, "STREAM_NAME|");
	if (filter & FILE_NOTIFY_CHANGE_STREAM_SIZE)
		result = talloc_asprintf_append(result, "STREAM_SIZE|");
	if (filter & FILE_NOTIFY_CHANGE_STREAM_WRITE)
		result = talloc_asprintf_append(result, "STREAM_WRITE|");

	if (result == NULL) return NULL;
	if (*result == '\0') return result;

	result[strlen(result)-1] = '\0';
	return result;
}

struct notify_change_event {
	struct timespec when;
	uint32_t action;
	const char *name;
}

struct notify_change_buf {
	/*
	 * Filters for reinitializing after notifyd has been restarted
	 */
	uint32_t filter;
	uint32_t subdir_filter;

	/*
	 * If no requests are pending, changes are queued here. Simple array,
	 * we only append.
	 */

	uint32_t max_buffer_size;

	/*
	 * num_changes == -1 means that we have got a catch-all change, when
	 * asked we just return NT_STATUS_OK without specific changes.
	 */
	int num_changes;
	struct notify_change_event *changes;

	/*
	 * If no changes are around requests are queued here. Using a linked
	 * list, because we have to append at the end and delete from the top.
	 */
	struct notify_change_request *requests;
}

struct notify_change_request {
	struct notify_change_request *prev, *next;
	struct files_struct *fsp;	/* backpointer for cancel by mid */
	struct smb_request *req;
	uint32_t filter;
	uint32_t max_param;
	void (*reply_fn)(struct smb_request *req,
			 NTSTATUS error_code,
			 uint8_t *buf, size_t len);
	struct notify_mid_map *mid_map;
	void *backend_data;
}

static void notify_fsp(files_struct *fsp, struct timespec when,
		       uint32_t action, const char *name);


void notify_callback(struct smbd_server_connection *sconn,
		     void *private_data, struct timespec when,
		     const struct notify_event *e)
{
	struct notify_fsp_state state = {
		.notified_fsp = private_data, .when = when, .e = e
	};
	files_forall(sconn, notify_fsp_cb, &state);
}


void ItemStackMetadata::serialize(std::ostream &os) const
{
	std::ostringstream os2;
	os2 << DESERIALIZE_START;
	for (const auto &stringvar : m_stringvars) {
		if (!stringvar.first.empty() || !stringvar.second.empty())
			os2 << stringvar.first << DESERIALIZE_KV_DELIM
				<< stringvar.second << DESERIALIZE_PAIR_DELIM;
	}
	os << serializeJsonStringIfNeeded(os2.str());
}

Sanitize ItemStack meta text

#define DESERIALIZE_START '\x01'

#define DESERIALIZE_KV_DELIM '\x02'

#define DESERIALIZE_PAIR_DELIM '\x03'


void ItemStackMetadata::clearToolCapabilities()
{
	setString(TOOLCAP_KEY, "");
}

Sanitize ItemStack meta text


static void sanitize_string(std::string &str)
{
	str.erase(std::remove(str.begin(), str.end(), DESERIALIZE_START), str.end());
	str.erase(std::remove(str.begin(), str.end(), DESERIALIZE_KV_DELIM), str.end());
	str.erase(std::remove(str.begin(), str.end(), DESERIALIZE_PAIR_DELIM), str.end());
}

Sanitize ItemStack meta text


void ItemStackMetadata::setToolCapabilities(const ToolCapabilities &caps)
{
	std::ostringstream os;
	caps.serializeJson(os);
	setString(TOOLCAP_KEY, os.str());
}

Sanitize ItemStack meta text


void ItemStackMetadata::updateToolCapabilities()
{
	if (contains(TOOLCAP_KEY)) {
		toolcaps_overridden = true;
		toolcaps_override = ToolCapabilities();
		std::istringstream is(getString(TOOLCAP_KEY));
		toolcaps_override.deserializeJson(is);
	} else {
		toolcaps_overridden = false;
	}
}

Sanitize ItemStack meta text


void ItemStackMetadata::clear()
{
	Metadata::clear();
	updateToolCapabilities();
}

Sanitize ItemStack meta text

#define TOOLCAP_KEY "tool_capabilities"

#define DESERIALIZE_START '\x01'

#define DESERIALIZE_START_STR "\x01"

#define DESERIALIZE_PAIR_DELIM '\x03'

#define DESERIALIZE_KV_DELIM '\x02'

#define DESERIALIZE_PAIR_DELIM_STR "\x03"

#define DESERIALIZE_KV_DELIM_STR "\x02"


void ItemStackMetadata::deSerialize(std::istream &is)
{
	std::string in = deSerializeJsonStringIfNeeded(is);

	m_stringvars.clear();

	if (!in.empty()) {
		if (in[0] == DESERIALIZE_START) {
			Strfnd fnd(in);
			fnd.to(1);
			while (!fnd.at_end()) {
				std::string name = fnd.next(DESERIALIZE_KV_DELIM_STR);
				std::string var  = fnd.next(DESERIALIZE_PAIR_DELIM_STR);
				m_stringvars[name] = var;
			}
		} else {
			// BACKWARDS COMPATIBILITY
			m_stringvars[""] = in;
		}
	}
	updateToolCapabilities();
}

Sanitize ItemStack meta text

#define DESERIALIZE_START '\x01'

#define DESERIALIZE_KV_DELIM_STR "\x02"

#define DESERIALIZE_PAIR_DELIM_STR "\x03"


bool ItemStackMetadata::setString(const std::string &name, const std::string &var)
{
	std::string clean_name = name;
	std::string clean_var = var;
	sanitize_string(clean_name);
	sanitize_string(clean_var);

	bool result = Metadata::setString(clean_name, clean_var);
	if (clean_name == TOOLCAP_KEY)
		updateToolCapabilities();
	return result;
}

Sanitize ItemStack meta text


static int init_cache_random_seq(struct kmem_cache *s)
{
	unsigned int count = oo_objects(s->oo);
	int err;

	/* Bailout if already initialised */
	if (s->random_seq)
		return 0;

	err = cache_random_seq_create(s, count, GFP_KERNEL);
	if (err) {
		pr_err("SLUB: Unable to initialize free list for %s\n",
			s->name);
		return err;
	}

	/* Transform to an offset on the set of pages */
	if (s->random_seq) {
		unsigned int i;

		for (i = 0; i < count; i++)
			s->random_seq[i] *= s->size;
	}
	return 0;
}

mm: slub: add missing TID bump in kmem_cache_alloc_bulk()

When kmem_cache_alloc_bulk() attempts to allocate N objects from a percpu
freelist of length M, and N > M > 0, it will first remove the M elements
from the percpu freelist, then call ___slab_alloc() to allocate the next
element and repopulate the percpu freelist. ___slab_alloc() can re-enable
IRQs via allocate_slab(), so the TID must be bumped before ___slab_alloc()
to properly commit the freelist head change.

Fix it by unconditionally bumping c->tid when entering the slowpath.

Cc: stable@vger.kernel.org
Fixes: ebe909e0fdb3 ("slub: improve bulk alloc strategy")
Signed-off-by: Jann Horn <jannh@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline unsigned long slabs_node(struct kmem_cache *s, int node)
							{ return 0; }


static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
{
	stat(s, CPUSLAB_FLUSH);
	deactivate_slab(s, c->page, c->freelist, c);

	c->tid = next_tid(c->tid);
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
{
	do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


void *__kmalloc(size_t size, gfp_t flags)
{
	struct kmem_cache *s;
	void *ret;

	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
		return kmalloc_large(size, flags);

	s = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

	ret = slab_alloc(s, flags, _RET_IP_);

	trace_kmalloc(_RET_IP_, ret, size, s->size, flags);

	ret = kasan_kmalloc(s, ret, size, flags);

	return ret;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static void set_cpu_partial(struct kmem_cache *s)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	/*
	 * cpu_partial determined the maximum number of objects kept in the
	 * per cpu partial lists of a processor.
	 *
	 * Per cpu partial lists mainly contain slabs that just have one
	 * object freed. If they are used for allocation then they can be
	 * filled up again with minimal effort. The slab will never hit the
	 * per node partial lists and therefore no locking will be required.
	 *
	 * This setting also determines
	 *
	 * A) The number of objects from per cpu partial slabs dumped to the
	 *    per node list when we reach the limit.
	 * B) The number of objects in cpu partial slabs to extract from the
	 *    per node list when we run out of per cpu objects. We only fetch
	 *    50% to keep some capacity around for frees.
	 */
	if (!kmem_cache_has_cpu_partial(s))
		s->cpu_partial = 0;
	else if (s->size >= PAGE_SIZE)
		s->cpu_partial = 2;
	else if (s->size >= 1024)
		s->cpu_partial = 6;
	else if (s->size >= 256)
		s->cpu_partial = 13;
	else
		s->cpu_partial = 30;
#endif
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static ssize_t object_size_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->object_size);
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t aliases_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
}


static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
	if (buf[0] == '1')
		kmem_cache_shrink_all(s);
	else
		return -EINVAL;
	return length;
}


static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	struct page *oldpage;
	int pages;
	int pobjects;

	preempt_disable();
	do {
		pages = 0;
		pobjects = 0;
		oldpage = this_cpu_read(s->cpu_slab->partial);

		if (oldpage) {
			pobjects = oldpage->pobjects;
			pages = oldpage->pages;
			if (drain && pobjects > s->cpu_partial) {
				unsigned long flags;
				/*
				 * partial array is full. Move the existing
				 * set to the per node partial list.
				 */
				local_irq_save(flags);
				unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
				local_irq_restore(flags);
				oldpage = NULL;
				pobjects = 0;
				pages = 0;
				stat(s, CPU_PARTIAL_DRAIN);
			}
		}

		pages++;
		pobjects += page->objects - page->inuse;

		page->pages = pages;
		page->pobjects = pobjects;
		page->next = oldpage;

	} while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
								!= oldpage);
	if (unlikely(!s->cpu_partial)) {
		unsigned long flags;

		local_irq_save(flags);
		unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
		local_irq_restore(flags);
	}
	preempt_enable();
#endif	/* CONFIG_SLUB_CPU_PARTIAL */
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
{
	void *start;
	void *cur;
	void *next;
	unsigned long idx, pos, page_limit, freelist_count;

	if (page->objects < 2 || !s->random_seq)
		return false;

	freelist_count = oo_objects(s->oo);
	pos = get_random_int() % freelist_count;

	page_limit = page->objects * s->size;
	start = fixup_red_left(s, page_address(page));

	/* First entry is used as the base of the freelist */
	cur = next_freelist_entry(s, page, &pos, start, page_limit,
				freelist_count);
	cur = setup_object(s, page, cur);
	page->freelist = cur;

	for (idx = 1; idx < page->objects; idx++) {
		next = next_freelist_entry(s, page, &pos, start, page_limit,
			freelist_count);
		next = setup_object(s, page, next);
		set_freepointer(s, cur, next);
		cur = next;
	}
	set_freepointer(s, cur, NULL);

	return true;
}


static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
{
	struct page *page;
	void *ptr = NULL;
	unsigned int order = get_order(size);

	flags |= __GFP_COMP;
	page = alloc_pages_node(node, flags, order);
	if (page) {
		ptr = page_address(page);
		mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE,
				    1 << order);
	}

	return kmalloc_large_node_hook(ptr, size, flags);
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

#define TID_STEP  roundup_pow_of_two(CONFIG_NR_CPUS)

#define TID_STEP 1

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static void flush_cpu_slab(void *d)
{
	struct kmem_cache *s = d;

	__flush_cpu_slab(s, smp_processor_id());
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


void *__kmalloc_node(size_t size, gfp_t flags, int node)
{
	struct kmem_cache *s;
	void *ret;

	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
		ret = kmalloc_large_node(size, flags, node);

		trace_kmalloc_node(_RET_IP_, ret,
				   size, PAGE_SIZE << get_order(size),
				   flags, node);

		return ret;
	}

	s = kmalloc_slab(size, flags);

	if (unlikely(ZERO_OR_NULL_PTR(s)))
		return s;

	ret = slab_alloc_node(s, flags, node, _RET_IP_);

	trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);

	ret = kasan_kmalloc(s, ret, size, flags);

	return ret;
}

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)

#define STAT_ATTR(si, text) 					\
static ssize_t text##_show(struct kmem_cache *s, char *buf)	\
{								\
	return show_stat(s, buf, si);				\
}								\
static ssize_t text##_store(struct kmem_cache *s,		\
				const char *buf, size_t length)	\
{								\
	if (buf[0] != '0')					\
		return -EINVAL;					\
	clear_stat(s, si);					\
	return length;						\
}								\
SLAB_ATTR(text);						\


static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
}


static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
	int objects = 0;
	int pages = 0;
	int cpu;
	int len;

	for_each_online_cpu(cpu) {
		struct page *page;

		page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));

		if (page) {
			pages += page->pages;
			objects += page->pobjects;
		}
	}

	len = sprintf(buf, "%d(%d)", objects, pages);

#ifdef CONFIG_SMP
	for_each_online_cpu(cpu) {
		struct page *page;

		page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));

		if (page && len < PAGE_SIZE - 20)
			len += sprintf(buf + len, " C%d=%d(%d)", cpu,
				page->pobjects, page->pages);
	}
#endif
	return len + sprintf(buf + len, "\n");
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static int list_locations(struct kmem_cache *s, char *buf,
					enum track_item alloc)
{
	int len = 0;
	unsigned long i;
	struct loc_track t = { 0, 0, NULL };
	int node;
	struct kmem_cache_node *n;

	if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location),
			     GFP_KERNEL)) {
		return sprintf(buf, "Out of memory\n");
	}
	/* Push back cpu slabs */
	flush_all(s);

	for_each_kmem_cache_node(s, node, n) {
		unsigned long flags;
		struct page *page;

		if (!atomic_long_read(&n->nr_slabs))
			continue;

		spin_lock_irqsave(&n->list_lock, flags);
		list_for_each_entry(page, &n->partial, slab_list)
			process_slab(&t, s, page, alloc);
		list_for_each_entry(page, &n->full, slab_list)
			process_slab(&t, s, page, alloc);
		spin_unlock_irqrestore(&n->list_lock, flags);
	}

	for (i = 0; i < t.count; i++) {
		struct location *l = &t.loc[i];

		if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100)
			break;
		len += sprintf(buf + len, "%7ld ", l->count);

		if (l->addr)
			len += sprintf(buf + len, "%pS", (void *)l->addr);
		else
			len += sprintf(buf + len, "<not-available>");

		if (l->sum_time != l->min_time) {
			len += sprintf(buf + len, " age=%ld/%ld/%ld",
				l->min_time,
				(long)div_u64(l->sum_time, l->count),
				l->max_time);
		} else
			len += sprintf(buf + len, " age=%ld",
				l->min_time);

		if (l->min_pid != l->max_pid)
			len += sprintf(buf + len, " pid=%ld-%ld",
				l->min_pid, l->max_pid);
		else
			len += sprintf(buf + len, " pid=%ld",
				l->min_pid);

		if (num_online_cpus() > 1 &&
				!cpumask_empty(to_cpumask(l->cpus)) &&
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " cpus=%*pbl",
					 cpumask_pr_args(to_cpumask(l->cpus)));

		if (nr_online_nodes > 1 && !nodes_empty(l->nodes) &&
				len < PAGE_SIZE - 60)
			len += scnprintf(buf + len, PAGE_SIZE - len - 50,
					 " nodes=%*pbl",
					 nodemask_pr_args(&l->nodes));

		len += sprintf(buf + len, "\n");
	}

	free_loc_track(&t);
	if (!t.count)
		len += sprintf(buf, "No data\n");
	return len;
}

#define MAX_OBJS_PER_PAGE	32767 /* since page.objects is u15 */

struct track {
	unsigned long addr;	/* Called from address */
#ifdef CONFIG_STACKTRACE
	unsigned long addrs[TRACK_ADDRS_COUNT];	/* Called from address */
#endif
	int cpu;		/* Was running on cpu */
	int pid;		/* Pid context */
	unsigned long when;	/* When did the operation occur */
}

enum track_item { TRACK_ALLOC, TRACK_FREE }

static int sysfs_slab_add(struct kmem_cache *);

static int sysfs_slab_alias(struct kmem_cache *, const char *);

static void memcg_propagate_slab_attrs(struct kmem_cache *s);

static void sysfs_slab_remove(struct kmem_cache *s);

#define for_each_object(__p, __s, __addr, __objects) \
	for (__p = fixup_red_left(__s, __addr); \
		__p < (__addr) + (__objects) * (__s)->size; \
		__p += (__s)->size)

static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];

static DEFINE_SPINLOCK(object_map_lock);

static slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;

static slab_flags_t slub_debug;

static char *slub_debug_slabs;

static int disable_higher_order_debug;

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
			const char *fmt, ...)
{
	va_list args;

char buf[100];

static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);

static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);

struct detached_freelist {
	struct page *page;
	void *tail;
	void *freelist;
	int cnt;
	struct kmem_cache *s;
}

static unsigned int slub_min_order;

static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;

static unsigned int slub_min_objects;

static struct kmem_cache *kmem_cache_node;

static struct notifier_block slab_memory_callback_nb = {
	.notifier_call = slab_memory_callback,
	.priority = SLAB_CALLBACK_PRI,
};

struct location {
	unsigned long count;
	unsigned long addr;
	long long sum_time;
	long min_time;
	long max_time;
	long min_pid;
	long max_pid;
	DECLARE_BITMAP(cpus, NR_CPUS);
	nodemask_t nodes;
}

struct loc_track {
	unsigned long max;
	unsigned long count;
	struct location *loc;
}

enum slab_stat_type {
	SL_ALL,			/* All slabs */
	SL_PARTIAL,		/* Only partially allocated slabs */
	SL_CPU,			/* Only slabs used for cpu caches */
	SL_OBJECTS,		/* Determine allocated objects not slabs */
	SL_TOTAL		/* Determine object capacity not slabs */
}

static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);

#define to_slab_attr(n) container_of(n, struct slab_attribute, attr)

#define to_slab(n) container_of(n, struct kmem_cache, kobj)

struct slab_attribute {
	struct attribute attr;
	ssize_t (*show)(struct kmem_cache *s, char *buf);
	ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
}

#define SLAB_ATTR_RO(_name) \
	static struct slab_attribute _name##_attr = \
	__ATTR(_name, 0400, _name##_show, NULL)

#define SLAB_ATTR(_name) \
	static struct slab_attribute _name##_attr =  \
	__ATTR(_name, 0600, _name##_show, _name##_store)


static void unfreeze_partials(struct kmem_cache *s,
		struct kmem_cache_cpu *c)
{
#ifdef CONFIG_SLUB_CPU_PARTIAL
	struct kmem_cache_node *n = NULL, *n2 = NULL;
	struct page *page, *discard_page = NULL;

	while ((page = c->partial)) {
		struct page new;
		struct page old;

		c->partial = page->next;

		n2 = get_node(s, page_to_nid(page));
		if (n != n2) {
			if (n)
				spin_unlock(&n->list_lock);

			n = n2;
			spin_lock(&n->list_lock);
		}

		do {

			old.freelist = page->freelist;
			old.counters = page->counters;
			VM_BUG_ON(!old.frozen);

			new.counters = old.counters;
			new.freelist = old.freelist;

			new.frozen = 0;

		} while (!__cmpxchg_double_slab(s, page,
				old.freelist, old.counters,
				new.freelist, new.counters,
				"unfreezing slab"));

		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
			page->next = discard_page;
			discard_page = page;
		} else {
			add_partial(n, page, DEACTIVATE_TO_TAIL);
			stat(s, FREE_ADD_PARTIAL);
		}
	}

	if (n)
		spin_unlock(&n->list_lock);

	while (discard_page) {
		page = discard_page;
		discard_page = discard_page->next;

		stat(s, DEACTIVATE_EMPTY);
		discard_slab(s, page);
		stat(s, FREE_SLAB);
	}
#endif	/* CONFIG_SLUB_CPU_PARTIAL */
}


void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
				    gfp_t gfpflags,
				    int node, size_t size)
{
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);

	trace_kmalloc_node(_RET_IP_, ret,
			   size, s->size, gfpflags, node);

	ret = kasan_kmalloc(s, ret, size, gfpflags);
	return ret;
}


static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
}


static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }