static int cbs_jpeg_write_unit(CodedBitstreamContext *ctx,
                                CodedBitstreamUnit *unit)
{
    CodedBitstreamJPEGContext *priv = ctx->priv_data;
    PutBitContext pbc;
    int err;

    if (!priv->write_buffer) {
        // Initial write buffer size is 1MB.
        priv->write_buffer_size = 1024 * 1024;

    reallocate_and_try_again:
        err = av_reallocp(&priv->write_buffer, priv->write_buffer_size);
        if (err < 0) {
            av_log(ctx->log_ctx, AV_LOG_ERROR, "Unable to allocate a "
                   "sufficiently large write buffer (last attempt "
                   "%"SIZE_SPECIFIER" bytes).\n", priv->write_buffer_size);
            return err;
        }
    }

    init_put_bits(&pbc, priv->write_buffer, priv->write_buffer_size);

    if (unit->type == JPEG_MARKER_SOS)
        err = cbs_jpeg_write_scan(ctx, unit, &pbc);
    else
        err = cbs_jpeg_write_segment(ctx, unit, &pbc);

    if (err == AVERROR(ENOSPC)) {
        // Overflow.
        priv->write_buffer_size *= 2;
        goto reallocate_and_try_again;
    }
    if (err < 0) {
        // Write failed for some other reason.
        return err;
    }

    if (put_bits_count(&pbc) % 8)
        unit->data_bit_padding = 8 - put_bits_count(&pbc) % 8;
    else
        unit->data_bit_padding = 0;

    unit->data_size = (put_bits_count(&pbc) + 7) / 8;
    flush_put_bits(&pbc);

    err = ff_cbs_alloc_unit_data(ctx, unit, unit->data_size);
    if (err < 0)
        return err;

    memcpy(unit->data, priv->write_buffer, unit->data_size);

    return 0;
}

#define HEADER(name) do { \
        ff_cbs_trace_header(ctx, name); \
    } while (0)

#define CHECK(call) do { \
        err = (call); \
        if (err < 0) \
            return err; \
    } while (0)

#define SUBSCRIPTS(subs, ...) (subs > 0 ? ((int[subs + 1]){ subs, __VA_ARGS__ }) : NULL)

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = range_min; \
        CHECK(ff_cbs_read_unsigned(ctx, rw, width, #name, \
                                   SUBSCRIPTS(subs, __VA_ARGS__), \
                                   &value, range_min, range_max)); \
        current->name = value; \
    } while (0)

#define READWRITE write

#define RWContext PutBitContext

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = current->name; \
        CHECK(ff_cbs_write_unsigned(ctx, rw, width, #name, \
                                    SUBSCRIPTS(subs, __VA_ARGS__), \
                                    value, range_min, range_max)); \
    } while (0)

const CodedBitstreamType ff_cbs_type_jpeg = {
    .codec_id          = AV_CODEC_ID_MJPEG,

    .priv_data_size    = sizeof(CodedBitstreamJPEGContext),

    .split_fragment    = &cbs_jpeg_split_fragment,
    .read_unit         = &cbs_jpeg_read_unit,
    .write_unit        = &cbs_jpeg_write_unit,
    .assemble_fragment = &cbs_jpeg_assemble_fragment,

    .close             = &cbs_jpeg_close,
};


static void cbs_jpeg_free_comment(void *unit, uint8_t *content)
{
    JPEGRawComment *comment = (JPEGRawComment*)content;
    av_buffer_unref(&comment->Cm_ref);
    av_freep(&content);
}

avcodec/cbs_jpeg: Check length for SOS

Fixes: out of array access
Fixes: 19734/clusterfuzz-testcase-minimized-ffmpeg_BSF_TRACE_HEADERS_fuzzer-5673507031875584
Fixes: 19353/clusterfuzz-testcase-minimized-ffmpeg_BSF_TRACE_HEADERS_fuzzer-5703944462663680

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 1812352d767ccf5431aa440123e2e260a4db2726)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

#define READ

#define WRITE

#define READWRITE read

#define READWRITE write

#define RWContext GetBitContext

#define RWContext PutBitContext

#define FUNC(name) cbs_jpeg_read_ ## name

#define FUNC(name) cbs_jpeg_write_ ## name

#define HEADER(name) do { \
        ff_cbs_trace_header(ctx, name); \
    } while (0)

#define u(width, name, range_min, range_max) \
    xu(width, name, range_min, range_max, 0)

#define SUBSCRIPTS(subs, ...) (subs > 0 ? ((int[subs + 1]){ subs, __VA_ARGS__ }) : NULL)

#define us(width, name, sub, range_min, range_max) \
    xu(width, name, range_min, range_max, 1, sub)

#define CHECK(call) do { \
        err = (call); \
        if (err < 0) \
            return err; \
    } while (0)

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = current->name; \
        CHECK(ff_cbs_write_unsigned(ctx, rw, width, #name, \
                                    SUBSCRIPTS(subs, __VA_ARGS__), \
                                    value, range_min, range_max)); \
    } while (0)

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = range_min; \
        CHECK(ff_cbs_read_unsigned(ctx, rw, width, #name, \
                                   SUBSCRIPTS(subs, __VA_ARGS__), \
                                   &value, range_min, range_max)); \
        current->name = value; \
    } while (0)

const CodedBitstreamType ff_cbs_type_jpeg = {
    .codec_id          = AV_CODEC_ID_MJPEG,

    .priv_data_size    = sizeof(CodedBitstreamJPEGContext),

    .split_fragment    = &cbs_jpeg_split_fragment,
    .read_unit         = &cbs_jpeg_read_unit,
    .write_unit        = &cbs_jpeg_write_unit,
    .assemble_fragment = &cbs_jpeg_assemble_fragment,

    .close             = &cbs_jpeg_close,
};


static int cbs_jpeg_split_fragment(CodedBitstreamContext *ctx,
                                   CodedBitstreamFragment *frag,
                                   int header)
{
    AVBufferRef *data_ref;
    uint8_t *data;
    size_t data_size;
    int unit, start, end, marker, next_start, next_marker;
    int err, i, j, length;

    if (frag->data_size < 4) {
        // Definitely too short to be meaningful.
        return AVERROR_INVALIDDATA;
    }

    for (i = 0; i + 1 < frag->data_size && frag->data[i] != 0xff; i++);
    if (i > 0) {
        av_log(ctx->log_ctx, AV_LOG_WARNING, "Discarding %d bytes at "
               "beginning of image.\n", i);
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size && frag->data[i]) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no SOI marker found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    if (marker != JPEG_MARKER_SOI) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: first "
               "marker is %02x, should be SOI.\n", marker);
        return AVERROR_INVALIDDATA;
    }
    for (++i; i + 1 < frag->data_size && frag->data[i] == 0xff; i++);
    if (i + 1 >= frag->data_size) {
        av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
               "no image content found.\n");
        return AVERROR_INVALIDDATA;
    }
    marker = frag->data[i];
    start  = i + 1;

    for (unit = 0;; unit++) {
        if (marker == JPEG_MARKER_EOI) {
            break;
        } else if (marker == JPEG_MARKER_SOS) {
            for (i = start; i + 1 < frag->data_size; i++) {
                if (frag->data[i] != 0xff)
                    continue;
                end = i;
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    if (frag->data[i] == 0x00)
                        continue;
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
                break;
            }
        } else {
            i = start;
            if (i + 2 > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            length = AV_RB16(frag->data + i);
            if (i + length > frag->data_size) {
                av_log(ctx->log_ctx, AV_LOG_ERROR, "Invalid JPEG image: "
                       "truncated at %02x marker segment.\n", marker);
                return AVERROR_INVALIDDATA;
            }
            end = start + length;

            i = end;
            if (frag->data[i] != 0xff) {
                next_marker = -1;
            } else {
                for (++i; i + 1 < frag->data_size &&
                          frag->data[i] == 0xff; i++);
                if (i + 1 >= frag->data_size) {
                    next_marker = -1;
                } else {
                    next_marker = frag->data[i];
                    next_start  = i + 1;
                }
            }
        }

        if (marker == JPEG_MARKER_SOS) {
            length = AV_RB16(frag->data + start);

            if (length > end - start)
                return AVERROR_INVALIDDATA;

            data_ref = NULL;
            data     = av_malloc(end - start +
                                 AV_INPUT_BUFFER_PADDING_SIZE);
            if (!data)
                return AVERROR(ENOMEM);

            memcpy(data, frag->data + start, length);
            for (i = start + length, j = length; i < end; i++, j++) {
                if (frag->data[i] == 0xff) {
                    while (frag->data[i] == 0xff)
                        ++i;
                    data[j] = 0xff;
                } else {
                    data[j] = frag->data[i];
                }
            }
            data_size = j;

            memset(data + data_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);

        } else {
            data      = frag->data + start;
            data_size = end - start;
            data_ref  = frag->data_ref;
        }

        err = ff_cbs_insert_unit_data(ctx, frag, unit, marker,
                                      data, data_size, data_ref);
        if (err < 0) {
            if (!data_ref)
                av_freep(&data);
            return err;
        }

        if (next_marker == -1)
            break;
        marker = next_marker;
        start  = next_start;
    }

    return 0;
}

#define HEADER(name) do { \
        ff_cbs_trace_header(ctx, name); \
    } while (0)

#define CHECK(call) do { \
        err = (call); \
        if (err < 0) \
            return err; \
    } while (0)

#define SUBSCRIPTS(subs, ...) (subs > 0 ? ((int[subs + 1]){ subs, __VA_ARGS__ }) : NULL)

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = range_min; \
        CHECK(ff_cbs_read_unsigned(ctx, rw, width, #name, \
                                   SUBSCRIPTS(subs, __VA_ARGS__), \
                                   &value, range_min, range_max)); \
        current->name = value; \
    } while (0)

#define xu(width, name, range_min, range_max, subs, ...) do { \
        uint32_t value = current->name; \
        CHECK(ff_cbs_write_unsigned(ctx, rw, width, #name, \
                                    SUBSCRIPTS(subs, __VA_ARGS__), \
                                    value, range_min, range_max)); \
    } while (0)

const CodedBitstreamType ff_cbs_type_jpeg = {
    .codec_id          = AV_CODEC_ID_MJPEG,

    .priv_data_size    = sizeof(CodedBitstreamJPEGContext),

    .split_fragment    = &cbs_jpeg_split_fragment,
    .read_unit         = &cbs_jpeg_read_unit,
    .write_unit        = &cbs_jpeg_write_unit,
    .assemble_fragment = &cbs_jpeg_assemble_fragment,

    .close             = &cbs_jpeg_close,
};


static void cbs_jpeg_free_application_data(void *unit, uint8_t *content)
{
    JPEGRawApplicationData *ad = (JPEGRawApplicationData*)content;
    av_buffer_unref(&ad->Ap_ref);
    av_freep(&content);
}


TfLiteStatus ResizeOutput(TfLiteContext* context, const TfLiteTensor* input,
                          const TfLiteTensor* axis, TfLiteTensor* output) {
  int axis_value;
  // Retrive all 8 bytes when axis type is kTfLiteInt64 to avoid data loss.
  if (axis->type == kTfLiteInt64) {
    axis_value = static_cast<int>(*GetTensorData<int64_t>(axis));
  } else {
    axis_value = *GetTensorData<int>(axis);
  }
  if (axis_value < 0) {
    axis_value += NumDimensions(input);
  }

  TF_LITE_ENSURE(context, axis_value >= 0);
  TF_LITE_ENSURE(context, axis_value < NumDimensions(input));

  // Copy the input dimensions to output except the axis dimension.
  TfLiteIntArray* output_dims = TfLiteIntArrayCreate(NumDimensions(input) - 1);
  int j = 0;
  for (int i = 0; i < NumDimensions(input); ++i) {
    if (i != axis_value) {
      output_dims->data[j] = SizeOfDimension(input, i);
      ++j;
    }
  }
  return context->ResizeTensor(context, output, output_dims);
}

Prevent array write out-of-bounds.

If user passes an invalid axis, then we copy one too many dimensions to the output in the loop below these checks. Even if we didn't do that, there will be further issues with an invalid axis, so we check for that right now.

PiperOrigin-RevId: 371023299
Change-Id: I9eca37ffc2b29e8e48710f500701270ef0790224

/* No explicit global definitions found. */


TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node, bool is_arg_max) {
  const TfLiteTensor* input;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kInputTensor, &input));
  const TfLiteTensor* axis;
  TF_LITE_ENSURE_OK(context, GetInputSafe(context, node, kAxis, &axis));
  TfLiteTensor* output;
  TF_LITE_ENSURE_OK(context,
                    GetOutputSafe(context, node, kOutputTensor, &output));
  if (IsDynamicTensor(output)) {
    TF_LITE_ENSURE_STATUS(ResizeOutput(context, input, axis, output));
  }

#define TF_LITE_ARG_MIN_MAX(data_type, axis_type, output_type) \
  optimized_ops::ArgMinMax(                                    \
      GetTensorShape(input), GetTensorData<data_type>(input),  \
      GetTensorData<axis_type>(axis), GetTensorShape(output),  \
      GetTensorData<output_type>(output), is_arg_max)
  if (axis->type == kTfLiteInt32) {
    switch (output->type) {
      case kTfLiteInt32: {
        switch (input->type) {
          case kTfLiteFloat32:
            TF_LITE_ARG_MIN_MAX(float, int32_t, int32_t);
            break;
          case kTfLiteUInt8:
            TF_LITE_ARG_MIN_MAX(uint8_t, int32_t, int32_t);
            break;
          case kTfLiteInt8:
            TF_LITE_ARG_MIN_MAX(int8_t, int32_t, int32_t);
            break;
          case kTfLiteInt32:
            TF_LITE_ARG_MIN_MAX(int32_t, int32_t, int32_t);
            break;
          default:
            context->ReportError(context,
                                 "Only float32, uint8, int8 and int32 are "
                                 "supported currently, got %s.",
                                 TfLiteTypeGetName(input->type));
            return kTfLiteError;
        }
      } break;
      case kTfLiteInt64: {
        switch (input->type) {
          case kTfLiteFloat32:
            TF_LITE_ARG_MIN_MAX(float, int32_t, int64_t);
            break;
          case kTfLiteUInt8:
            TF_LITE_ARG_MIN_MAX(uint8_t, int32_t, int64_t);
            break;
          case kTfLiteInt8:
            TF_LITE_ARG_MIN_MAX(int8_t, int32_t, int64_t);
            break;
          case kTfLiteInt32:
            TF_LITE_ARG_MIN_MAX(int32_t, int32_t, int64_t);
            break;
          default:
            context->ReportError(context,
                                 "Only float32, uint8, int8 and int32 are "
                                 "supported currently, got %s.",
                                 TfLiteTypeGetName(input->type));
            return kTfLiteError;
        }
      } break;
      default:
        context->ReportError(
            context, "Only int32 and int64 are supported currently, got %s.",
            TfLiteTypeGetName(output->type));
        return kTfLiteError;
    }
  } else {
    switch (output->type) {
      case kTfLiteInt32: {
        switch (input->type) {
          case kTfLiteFloat32:
            TF_LITE_ARG_MIN_MAX(float, int64_t, int32_t);
            break;
          case kTfLiteUInt8:
            TF_LITE_ARG_MIN_MAX(uint8_t, int64_t, int32_t);
            break;
          case kTfLiteInt8:
            TF_LITE_ARG_MIN_MAX(int8_t, int64_t, int32_t);
            break;
          case kTfLiteInt32:
            TF_LITE_ARG_MIN_MAX(int32_t, int64_t, int32_t);
            break;
          default:
            context->ReportError(context,
                                 "Only float32, uint8, int8 and int32 are "
                                 "supported currently, got %s.",
                                 TfLiteTypeGetName(input->type));
            return kTfLiteError;
        }
      } break;
      case kTfLiteInt64: {
        switch (input->type) {
          case kTfLiteFloat32:
            TF_LITE_ARG_MIN_MAX(float, int64_t, int64_t);
            break;
          case kTfLiteUInt8:
            TF_LITE_ARG_MIN_MAX(uint8_t, int64_t, int64_t);
            break;
          case kTfLiteInt8:
            TF_LITE_ARG_MIN_MAX(int8_t, int64_t, int64_t);
            break;
          case kTfLiteInt32:
            TF_LITE_ARG_MIN_MAX(int32_t, int64_t, int64_t);
            break;
          default:
            context->ReportError(context,
                                 "Only float32, uint8, int8 and int32 are "
                                 "supported currently, got %s.",
                                 TfLiteTypeGetName(input->type));
            return kTfLiteError;
        }
      } break;
      default:
        context->ReportError(
            context, "Only int32 and int64 are supported currently, got %s.",
            TfLiteTypeGetName(output->type));
        return kTfLiteError;
    }
  }
#undef TF_LITE_ARG_MIN_MAX

  return kTfLiteOk;
}


link_heap_page(mrb_gc *gc, mrb_heap_page *page)
{
  page->next = gc->heaps;
  if (gc->heaps)
    gc->heaps->prev = page;
  gc->heaps = page;
}

Cancel 9cdf439

Should not free the pointer in `realloc` since it can cause
use-after-free problem.


gc_gray_counts(mrb_state *mrb, mrb_gc *gc, struct RBasic *obj)
{
  size_t children = 0;

  switch (obj->tt) {
  case MRB_TT_ICLASS:
    children++;
    break;

  case MRB_TT_CLASS:
  case MRB_TT_SCLASS:
  case MRB_TT_MODULE:
    {
      struct RClass *c = (struct RClass*)obj;

      children += mrb_gc_mark_iv_size(mrb, (struct RObject*)obj);
      children += mrb_gc_mark_mt_size(mrb, c);
      children++;
    }
    break;

  case MRB_TT_OBJECT:
  case MRB_TT_DATA:
  case MRB_TT_EXCEPTION:
    children += mrb_gc_mark_iv_size(mrb, (struct RObject*)obj);
    break;

  case MRB_TT_ENV:
    children += MRB_ENV_LEN(obj);
    break;

  case MRB_TT_FIBER:
    {
      struct mrb_context *c = ((struct RFiber*)obj)->cxt;
      size_t i;
      mrb_callinfo *ci;

      if (!c || c->status == MRB_FIBER_TERMINATED) break;

      /* mark stack */
      i = c->stack - c->stbase;

      if (c->ci) {
        i += ci_nregs(c->ci);
      }
      if (c->stbase + i > c->stend) i = c->stend - c->stbase;
      children += i;

      /* mark ensure stack */
      children += c->eidx;

      /* mark closure */
      if (c->cibase) {
        for (i=0, ci = c->cibase; ci <= c->ci; i++, ci++)
          ;
      }
      children += i;
    }
    break;

  case MRB_TT_ARRAY:
    {
      struct RArray *a = (struct RArray*)obj;
      children += ARY_LEN(a);
    }
    break;

  case MRB_TT_HASH:
    children += mrb_gc_mark_iv_size(mrb, (struct RObject*)obj);
    children += mrb_gc_mark_hash_size(mrb, (struct RHash*)obj);
    break;

  case MRB_TT_PROC:
  case MRB_TT_RANGE:
    children+=2;
    break;

  default:
    break;
  }
  return children;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_malloc_simple(mrb_state *mrb, size_t len)
{
  return mrb_realloc_simple(mrb, 0, len);
}


mrb_alloca(mrb_state *mrb, size_t size)
{
  struct RString *s;
  s = (struct RString*)mrb_obj_alloc(mrb, MRB_TT_STRING, mrb->string_class);
  return s->as.heap.ptr = (char*)mrb_malloc(mrb, size);
}


mrb_free(mrb_state *mrb, void *p)
{
  (mrb->allocf)(mrb, p, 0, mrb->allocf_ud);
}


mrb_gc_unregister(mrb_state *mrb, mrb_value obj)
{
  mrb_sym root;
  mrb_value table;
  struct RArray *a;
  mrb_int i;

  if (mrb_immediate_p(obj)) return;
  root = mrb_intern_lit(mrb, GC_ROOT_NAME);
  table = mrb_gv_get(mrb, root);
  if (mrb_nil_p(table)) return;
  if (!mrb_array_p(table)) {
    mrb_gv_set(mrb, root, mrb_nil_value());
    return;
  }
  a = mrb_ary_ptr(table);
  mrb_ary_modify(mrb, a);
  for (i = 0; i < ARY_LEN(a); i++) {
    if (mrb_ptr(ARY_PTR(a)[i]) == mrb_ptr(obj)) {
      mrb_int len = ARY_LEN(a)-1;
      mrb_value *ptr = ARY_PTR(a);

      ARY_SET_LEN(a, len);
      memmove(&ptr[i], &ptr[i + 1], (len - i) * sizeof(mrb_value));
      break;
    }
  }
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

#define GC_ROOT_NAME "_gc_root_"

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


prepare_incremental_sweep(mrb_state *mrb, mrb_gc *gc)
{
  gc->state = MRB_GC_STATE_SWEEP;
  gc->sweeps = gc->heaps;
  gc->live_after_mark = gc->live;
}


incremental_gc_until(mrb_state *mrb, mrb_gc *gc, mrb_gc_state to_state)
{
  do {
    incremental_gc(mrb, gc, SIZE_MAX);
  } while (gc->state != to_state);
}


gc_each_objects(mrb_state *mrb, mrb_gc *gc, mrb_each_object_callback *callback, void *data)
{
  mrb_heap_page* page;

  page = gc->heaps;
  while (page != NULL) {
    RVALUE *p;
    int i;

    p = objects(page);
    for (i=0; i < MRB_HEAP_PAGE_SIZE; i++) {
      if ((*callback)(mrb, &p[i].as.basic, data) == MRB_EACH_OBJ_BREAK)
        return;
    }
    page = page->next;
  }
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define MRB_HEAP_PAGE_SIZE 1024

#define objects(p) ((RVALUE *)p->objects)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

static mrb_value gc_test(mrb_state *, mrb_value);


heap_p(mrb_gc *gc, struct RBasic *object)
{
  mrb_heap_page* page;

  page = gc->heaps;
  while (page) {
    RVALUE *p;

    p = objects(page);
    if (&p[0].as.basic <= object && object <= &p[MRB_HEAP_PAGE_SIZE].as.basic) {
      return TRUE;
    }
    page = page->next;
  }
  return FALSE;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define MRB_HEAP_PAGE_SIZE 1024

#define objects(p) ((RVALUE *)p->objects)

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct


link_free_heap_page(mrb_gc *gc, mrb_heap_page *page)
{
  page->free_next = gc->free_heaps;
  if (gc->free_heaps) {
    gc->free_heaps->free_prev = page;
  }
  gc->free_heaps = page;
}


gc_start(mrb_state *mrb, mrb_value obj)
{
  mrb_full_gc(mrb);
  return mrb_nil_value();
}


mrb_gc_protect(mrb_state *mrb, mrb_value obj)
{
  if (mrb_immediate_p(obj)) return;
  gc_protect(mrb, &mrb->gc, mrb_basic_ptr(obj));
}


gc_disable(mrb_state *mrb, mrb_value obj)
{
  mrb_bool old = mrb->gc.disabled;

  mrb->gc.disabled = TRUE;

  return mrb_bool_value(old);
}


free_heap(mrb_state *mrb, mrb_gc *gc)
{
  mrb_heap_page *page = gc->heaps;
  mrb_heap_page *tmp;
  RVALUE *p, *e;

  while (page) {
    tmp = page;
    page = page->next;
    for (p = objects(tmp), e=p+MRB_HEAP_PAGE_SIZE; p<e; p++) {
      if (p->as.free.tt != MRB_TT_FREE)
        obj_free(mrb, &p->as.basic, TRUE);
    }
    mrb_free(mrb, tmp);
  }
}


ci_nregs(mrb_callinfo *ci)
{
  struct RProc *p = ci->proc;
  int n = 0;

  if (!p) {
    if (ci->argc < 0) return 3;
    return ci->argc+2;
  }
  if (!MRB_PROC_CFUNC_P(p) && p->body.irep) {
    n = p->body.irep->nregs;
  }
  if (ci->argc < 0) {
    if (n < 3) n = 3; /* self + args + blk */
  }
  if (ci->argc > n) {
    n = ci->argc + 2; /* self + blk */
  }
  return n;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct


mrb_gc_init(mrb_state *mrb, mrb_gc *gc)
{
#ifndef MRB_GC_FIXED_ARENA
  gc->arena = (struct RBasic**)mrb_malloc(mrb, sizeof(struct RBasic*)*MRB_GC_ARENA_SIZE);
  gc->arena_capa = MRB_GC_ARENA_SIZE;
#endif

  gc->current_white_part = GC_WHITE_A;
  gc->heaps = NULL;
  gc->free_heaps = NULL;
  add_heap(mrb, gc);
  gc->interval_ratio = DEFAULT_GC_INTERVAL_RATIO;
  gc->step_ratio = DEFAULT_GC_STEP_RATIO;
#ifndef MRB_GC_TURN_OFF_GENERATIONAL
  gc->generational = TRUE;
  gc->full = TRUE;
#endif

#ifdef GC_PROFILE
  program_invoke_time = gettimeofday_time();
#endif
}


mrb_calloc(mrb_state *mrb, size_t nelem, size_t len)
{
  void *p;

  if (nelem > 0 && len > 0 &&
      nelem <= SIZE_MAX / len) {
    size_t size;
    size = nelem * len;
    p = mrb_malloc(mrb, size);

    memset(p, 0, size);
  }
  else {
    p = NULL;
  }

  return p;
}


gc_step_ratio_set(mrb_state *mrb, mrb_value obj)
{
  mrb_int ratio;

  mrb_get_args(mrb, "i", &ratio);
  mrb->gc.step_ratio = (int)ratio;
  return mrb_nil_value();
}

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

static mrb_value gc_test(mrb_state *, mrb_value);


change_gen_gc_mode(mrb_state *mrb, mrb_gc *gc, mrb_bool enable)
{
  if (gc->disabled || gc->iterating) {
    mrb_raise(mrb, E_RUNTIME_ERROR, "generational mode changed when GC disabled");
    return;
  }
  if (is_generational(gc) && !enable) {
    clear_all_old(mrb, gc);
    mrb_assert(gc->state == MRB_GC_STATE_ROOT);
    gc->full = FALSE;
  }
  else if (!is_generational(gc) && enable) {
    incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
    gc->majorgc_old_threshold = gc->live_after_mark/100 * MAJOR_GC_INC_RATIO;
    gc->full = FALSE;
  }
  gc->generational = enable;
}

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

#define MAJOR_GC_INC_RATIO 120

#define is_generational(gc) ((gc)->generational)

#define is_major_gc(gc) (is_generational(gc) && (gc)->full)

#define is_minor_gc(gc) (is_generational(gc) && !(gc)->full)

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

static mrb_value gc_test(mrb_state *, mrb_value);


incremental_gc(mrb_state *mrb, mrb_gc *gc, size_t limit)
{
  switch (gc->state) {
  case MRB_GC_STATE_ROOT:
    root_scan_phase(mrb, gc);
    gc->state = MRB_GC_STATE_MARK;
    flip_white_part(gc);
    return 0;
  case MRB_GC_STATE_MARK:
    if (gc->gray_list) {
      return incremental_marking_phase(mrb, gc, limit);
    }
    else {
      final_marking_phase(mrb, gc);
      prepare_incremental_sweep(mrb, gc);
      return 0;
    }
  case MRB_GC_STATE_SWEEP: {
     size_t tried_sweep = 0;
     tried_sweep = incremental_sweep_phase(mrb, gc, limit);
     if (tried_sweep == 0)
       gc->state = MRB_GC_STATE_ROOT;
     return tried_sweep;
  }
  default:
    /* unknown state */
    mrb_assert(0);
    return 0;
  }
}


mrb_malloc(mrb_state *mrb, size_t len)
{
  return mrb_realloc(mrb, 0, len);
}


mrb_gc_register(mrb_state *mrb, mrb_value obj)
{
  mrb_sym root;
  mrb_value table;

  if (mrb_immediate_p(obj)) return;
  root = mrb_intern_lit(mrb, GC_ROOT_NAME);
  table = mrb_gv_get(mrb, root);
  if (mrb_nil_p(table) || !mrb_array_p(table)) {
    table = mrb_ary_new(mrb);
    mrb_gv_set(mrb, root, table);
  }
  mrb_ary_push(mrb, table, obj);
}

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

#define GC_ROOT_NAME "_gc_root_"

static mrb_value gc_test(mrb_state *, mrb_value);


gc_mark_gray_list(mrb_state *mrb, mrb_gc *gc) {
  while (gc->gray_list) {
    if (is_gray(gc->gray_list))
      gc_mark_children(mrb, gc, gc->gray_list);
    else
      gc->gray_list = gc->gray_list->gcnext;
  }
}


obj_free(mrb_state *mrb, struct RBasic *obj, int end)
{
  DEBUG(fprintf(stderr, "obj_free(%p,tt=%d)\n",obj,obj->tt));
  switch (obj->tt) {
    /* immediate - no mark */
  case MRB_TT_TRUE:
  case MRB_TT_FIXNUM:
  case MRB_TT_SYMBOL:
    /* cannot happen */
    return;

#ifndef MRB_WITHOUT_FLOAT
  case MRB_TT_FLOAT:
#ifdef MRB_WORD_BOXING
    break;
#else
    return;
#endif
#endif

  case MRB_TT_OBJECT:
    mrb_gc_free_iv(mrb, (struct RObject*)obj);
    break;

  case MRB_TT_EXCEPTION:
    mrb_gc_free_iv(mrb, (struct RObject*)obj);
    break;

  case MRB_TT_CLASS:
  case MRB_TT_MODULE:
  case MRB_TT_SCLASS:
    mrb_gc_free_mt(mrb, (struct RClass*)obj);
    mrb_gc_free_iv(mrb, (struct RObject*)obj);
    mrb_mc_clear_by_class(mrb, (struct RClass*)obj);
    break;
  case MRB_TT_ICLASS:
    if (MRB_FLAG_TEST(obj, MRB_FL_CLASS_IS_ORIGIN))
      mrb_gc_free_mt(mrb, (struct RClass*)obj);
    mrb_mc_clear_by_class(mrb, (struct RClass*)obj);
    break;
  case MRB_TT_ENV:
    {
      struct REnv *e = (struct REnv*)obj;

      if (MRB_ENV_ONSTACK_P(e)) {
        /* cannot be freed */
        e->stack = NULL;
        break;
      }
      mrb_free(mrb, e->stack);
      e->stack = NULL;
    }
    break;

  case MRB_TT_FIBER:
    {
      struct mrb_context *c = ((struct RFiber*)obj)->cxt;

      if (c && c != mrb->root_c) {
        if (!end && c->status != MRB_FIBER_TERMINATED) {
          mrb_callinfo *ci = c->ci;
          mrb_callinfo *ce = c->cibase;

          while (ce <= ci) {
            struct REnv *e = ci->env;
            if (e && !mrb_object_dead_p(mrb, (struct RBasic*)e) &&
                e->tt == MRB_TT_ENV && MRB_ENV_ONSTACK_P(e)) {
              mrb_env_unshare(mrb, e);
            }
            ci--;
          }
        }
        mrb_free_context(mrb, c);
      }
    }
    break;

  case MRB_TT_ARRAY:
    if (ARY_SHARED_P(obj))
      mrb_ary_decref(mrb, ((struct RArray*)obj)->as.heap.aux.shared);
    else if (!ARY_EMBED_P(obj))
      mrb_free(mrb, ((struct RArray*)obj)->as.heap.ptr);
    break;

  case MRB_TT_HASH:
    mrb_gc_free_iv(mrb, (struct RObject*)obj);
    mrb_gc_free_hash(mrb, (struct RHash*)obj);
    break;

  case MRB_TT_STRING:
    mrb_gc_free_str(mrb, (struct RString*)obj);
    break;

  case MRB_TT_PROC:
    {
      struct RProc *p = (struct RProc*)obj;

      if (!MRB_PROC_CFUNC_P(p) && p->body.irep) {
        mrb_irep *irep = p->body.irep;
        if (end) {
          mrb_irep_cutref(mrb, irep);
        }
        mrb_irep_decref(mrb, irep);
      }
    }
    break;

  case MRB_TT_RANGE:
    mrb_gc_free_range(mrb, ((struct RRange*)obj));
    break;

  case MRB_TT_DATA:
    {
      struct RData *d = (struct RData*)obj;
      if (d->type && d->type->dfree) {
        d->type->dfree(mrb, d->data);
      }
      mrb_gc_free_iv(mrb, (struct RObject*)obj);
    }
    break;

  default:
    break;
  }
  obj->tt = MRB_TT_FREE;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define DEBUG(x) (x)

#define DEBUG(x)

#define is_dead(s, o) (((o)->color & other_white_part(s) & GC_WHITES) || (o)->tt == MRB_TT_FREE)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


incremental_gc_step(mrb_state *mrb, mrb_gc *gc)
{
  size_t limit = 0, result = 0;
  limit = (GC_STEP_SIZE/100) * gc->step_ratio;
  while (result < limit) {
    result += incremental_gc(mrb, gc, limit);
    if (gc->state == MRB_GC_STATE_ROOT)
      break;
  }

  gc->threshold = gc->live + GC_STEP_SIZE;
}


gc_interval_ratio_get(mrb_state *mrb, mrb_value obj)
{
  return mrb_fixnum_value(mrb->gc.interval_ratio);
}


gc_step_ratio_get(mrb_state *mrb, mrb_value obj)
{
  return mrb_fixnum_value(mrb->gc.step_ratio);
}


add_gray_list(mrb_state *mrb, mrb_gc *gc, struct RBasic *obj)
{
#ifdef MRB_GC_STRESS
  if (obj->tt > MRB_TT_MAXDEFINE) {
    abort();
  }
#endif
  paint_gray(obj);
  obj->gcnext = gc->gray_list;
  gc->gray_list = obj;
}


unlink_heap_page(mrb_gc *gc, mrb_heap_page *page)
{
  if (page->prev)
    page->prev->next = page->next;
  if (page->next)
    page->next->prev = page->prev;
  if (gc->heaps == page)
    gc->heaps = page->next;
  page->prev = NULL;
  page->next = NULL;
}


unlink_free_heap_page(mrb_gc *gc, mrb_heap_page *page)
{
  if (page->free_prev)
    page->free_prev->free_next = page->free_next;
  if (page->free_next)
    page->free_next->free_prev = page->free_prev;
  if (gc->free_heaps == page)
    gc->free_heaps = page->free_next;
  page->free_prev = NULL;
  page->free_next = NULL;
}


mrb_object_dead_p(mrb_state *mrb, struct RBasic *object) {
  mrb_gc *gc = &mrb->gc;
  if (!heap_p(gc, object)) return TRUE;
  return is_dead(gc, object);
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define is_dead(s, o) (((o)->color & other_white_part(s) & GC_WHITES) || (o)->tt == MRB_TT_FREE)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_gc_mark(mrb_state *mrb, struct RBasic *obj)
{
  if (obj == 0) return;
  if (!is_white(obj)) return;
  mrb_assert((obj)->tt != MRB_TT_FREE);
  add_gray_list(mrb, &mrb->gc, obj);
}


mrb_field_write_barrier(mrb_state *mrb, struct RBasic *obj, struct RBasic *value)
{
  mrb_gc *gc = &mrb->gc;

  if (!is_black(obj)) return;
  if (!is_white(value)) return;

  mrb_assert(gc->state == MRB_GC_STATE_MARK || (!is_dead(gc, value) && !is_dead(gc, obj)));
  mrb_assert(is_generational(gc) || gc->state != MRB_GC_STATE_ROOT);

  if (is_generational(gc) || gc->state == MRB_GC_STATE_MARK) {
    add_gray_list(mrb, gc, value);
  }
  else {
    mrb_assert(gc->state == MRB_GC_STATE_SWEEP);
    paint_partial_white(gc, obj); /* for never write barriers */
  }
}


mrb_init_gc(mrb_state *mrb)
{
  struct RClass *gc;

  mrb_static_assert(sizeof(RVALUE) <= sizeof(void*) * 6,
                    "RVALUE size must be within 6 words");

  gc = mrb_define_module(mrb, "GC");

  mrb_define_class_method(mrb, gc, "start", gc_start, MRB_ARGS_NONE());
  mrb_define_class_method(mrb, gc, "enable", gc_enable, MRB_ARGS_NONE());
  mrb_define_class_method(mrb, gc, "disable", gc_disable, MRB_ARGS_NONE());
  mrb_define_class_method(mrb, gc, "interval_ratio", gc_interval_ratio_get, MRB_ARGS_NONE());
  mrb_define_class_method(mrb, gc, "interval_ratio=", gc_interval_ratio_set, MRB_ARGS_REQ(1));
  mrb_define_class_method(mrb, gc, "step_ratio", gc_step_ratio_get, MRB_ARGS_NONE());
  mrb_define_class_method(mrb, gc, "step_ratio=", gc_step_ratio_set, MRB_ARGS_REQ(1));
  mrb_define_class_method(mrb, gc, "generational_mode=", gc_generational_mode_set, MRB_ARGS_REQ(1));
  mrb_define_class_method(mrb, gc, "generational_mode", gc_generational_mode_get, MRB_ARGS_NONE());
#ifdef GC_TEST
#ifdef GC_DEBUG
  mrb_define_class_method(mrb, gc, "test", gc_test, MRB_ARGS_NONE());
#endif
#endif
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define objects(p) ((RVALUE *)p->objects)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


root_scan_phase(mrb_state *mrb, mrb_gc *gc)
{
  int i, e;

  if (!is_minor_gc(gc)) {
    gc->gray_list = NULL;
    gc->atomic_gray_list = NULL;
  }

  mrb_gc_mark_gv(mrb);
  /* mark arena */
  for (i=0,e=gc->arena_idx; i<e; i++) {
    mrb_gc_mark(mrb, gc->arena[i]);
  }
  /* mark class hierarchy */
  mrb_gc_mark(mrb, (struct RBasic*)mrb->object_class);

  /* mark built-in classes */
  mrb_gc_mark(mrb, (struct RBasic*)mrb->class_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->module_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->proc_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->string_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->array_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->hash_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->range_class);

#ifndef MRB_WITHOUT_FLOAT
  mrb_gc_mark(mrb, (struct RBasic*)mrb->float_class);
#endif
  mrb_gc_mark(mrb, (struct RBasic*)mrb->fixnum_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->true_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->false_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->nil_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->symbol_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->kernel_module);

  mrb_gc_mark(mrb, (struct RBasic*)mrb->eException_class);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->eStandardError_class);

  /* mark top_self */
  mrb_gc_mark(mrb, (struct RBasic*)mrb->top_self);
  /* mark exception */
  mrb_gc_mark(mrb, (struct RBasic*)mrb->exc);
  /* mark pre-allocated exception */
  mrb_gc_mark(mrb, (struct RBasic*)mrb->nomem_err);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->stack_err);
#ifdef MRB_GC_FIXED_ARENA
  mrb_gc_mark(mrb, (struct RBasic*)mrb->arena_err);
#endif

  mark_context(mrb, mrb->c);
  if (mrb->root_c != mrb->c) {
    mark_context(mrb, mrb->root_c);
  }
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

#define is_minor_gc(gc) (is_generational(gc) && !(gc)->full)

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_objspace_each_objects(mrb_state *mrb, mrb_each_object_callback *callback, void *data)
{
  mrb_bool iterating = mrb->gc.iterating;

  mrb_full_gc(mrb);
  mrb->gc.iterating = TRUE;
  if (iterating) {
    gc_each_objects(mrb, &mrb->gc, callback, data);
  }
  else {
    struct mrb_jmpbuf *prev_jmp = mrb->jmp;
    struct mrb_jmpbuf c_jmp;

    MRB_TRY(&c_jmp) {
      mrb->jmp = &c_jmp;
      gc_each_objects(mrb, &mrb->gc, callback, data);
      mrb->jmp = prev_jmp;
      mrb->gc.iterating = iterating;
   } MRB_CATCH(&c_jmp) {
      mrb->gc.iterating = iterating;
      mrb->jmp = prev_jmp;
      MRB_THROW(prev_jmp);
    } MRB_END_EXC(&c_jmp);
  }
}


incremental_sweep_phase(mrb_state *mrb, mrb_gc *gc, size_t limit)
{
  mrb_heap_page *page = gc->sweeps;
  size_t tried_sweep = 0;

  while (page && (tried_sweep < limit)) {
    RVALUE *p = objects(page);
    RVALUE *e = p + MRB_HEAP_PAGE_SIZE;
    size_t freed = 0;
    mrb_bool dead_slot = TRUE;
    mrb_bool full = (page->freelist == NULL);

    if (is_minor_gc(gc) && page->old) {
      /* skip a slot which doesn't contain any young object */
      p = e;
      dead_slot = FALSE;
    }
    while (p<e) {
      if (is_dead(gc, &p->as.basic)) {
        if (p->as.basic.tt != MRB_TT_FREE) {
          obj_free(mrb, &p->as.basic, FALSE);
          if (p->as.basic.tt == MRB_TT_FREE) {
            p->as.free.next = page->freelist;
            page->freelist = (struct RBasic*)p;
            freed++;
          }
          else {
            dead_slot = FALSE;
          }
        }
      }
      else {
        if (!is_generational(gc))
          paint_partial_white(gc, &p->as.basic); /* next gc target */
        dead_slot = FALSE;
      }
      p++;
    }

    /* free dead slot */
    if (dead_slot && freed < MRB_HEAP_PAGE_SIZE) {
      mrb_heap_page *next = page->next;

      unlink_heap_page(gc, page);
      unlink_free_heap_page(gc, page);
      mrb_free(mrb, page);
      page = next;
    }
    else {
      if (full && freed > 0) {
        link_free_heap_page(gc, page);
      }
      if (page->freelist == NULL && is_minor_gc(gc))
        page->old = TRUE;
      else
        page->old = FALSE;
      page = page->next;
    }
    tried_sweep += MRB_HEAP_PAGE_SIZE;
    gc->live -= freed;
    gc->live_after_mark -= freed;
  }
  gc->sweeps = page;
  return tried_sweep;
}


mrb_full_gc(mrb_state *mrb)
{
  mrb_gc *gc = &mrb->gc;

  if (!mrb->c) return;
  if (gc->disabled || gc->iterating) return;

  GC_INVOKE_TIME_REPORT("mrb_full_gc()");
  GC_TIME_START;

  if (is_generational(gc)) {
    /* clear all the old objects back to young */
    clear_all_old(mrb, gc);
    gc->full = TRUE;
  }
  else if (gc->state != MRB_GC_STATE_ROOT) {
    /* finish half baked GC cycle */
    incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
  }

  incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
  gc->threshold = (gc->live_after_mark/100) * gc->interval_ratio;

  if (is_generational(gc)) {
    gc->majorgc_old_threshold = gc->live_after_mark/100 * MAJOR_GC_INC_RATIO;
    gc->full = FALSE;
  }

  GC_TIME_STOP_AND_REPORT;
}

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define GC_INVOKE_TIME_REPORT(s)

#define GC_TIME_START

#define GC_TIME_STOP_AND_REPORT

#define objects(p) ((RVALUE *)p->objects)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

#define MAJOR_GC_INC_RATIO 120

#define is_generational(gc) ((gc)->generational)

#define is_major_gc(gc) (is_generational(gc) && (gc)->full)

#define is_minor_gc(gc) (is_generational(gc) && !(gc)->full)

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

static mrb_value gc_test(mrb_state *, mrb_value);


gc_generational_mode_set(mrb_state *mrb, mrb_value self)
{
  mrb_bool enable;

  mrb_get_args(mrb, "b", &enable);
  if (mrb->gc.generational != enable)
    change_gen_gc_mode(mrb, &mrb->gc, enable);

  return mrb_bool_value(enable);
}


mrb_gc_destroy(mrb_state *mrb, mrb_gc *gc)
{
  free_heap(mrb, gc);
#ifndef MRB_GC_FIXED_ARENA
  mrb_free(mrb, gc->arena);
#endif
}


mrb_obj_alloc(mrb_state *mrb, enum mrb_vtype ttype, struct RClass *cls)
{
  struct RBasic *p;
  static const RVALUE RVALUE_zero = { { { NULL, NULL, MRB_TT_FALSE } } };
  mrb_gc *gc = &mrb->gc;

  if (cls) {
    enum mrb_vtype tt;

    switch (cls->tt) {
    case MRB_TT_CLASS:
    case MRB_TT_SCLASS:
    case MRB_TT_MODULE:
    case MRB_TT_ENV:
      break;
    default:
      mrb_raise(mrb, E_TYPE_ERROR, "allocation failure");
    }
    tt = MRB_INSTANCE_TT(cls);
    if (tt != MRB_TT_FALSE &&
        ttype != MRB_TT_SCLASS &&
        ttype != MRB_TT_ICLASS &&
        ttype != MRB_TT_ENV &&
        ttype != tt) {
      mrb_raisef(mrb, E_TYPE_ERROR, "allocation failure of %C", cls);
    }
  }

#ifdef MRB_GC_STRESS
  mrb_full_gc(mrb);
#endif
  if (gc->threshold < gc->live) {
    mrb_incremental_gc(mrb);
  }
  if (gc->free_heaps == NULL) {
    add_heap(mrb, gc);
  }

  p = gc->free_heaps->freelist;
  gc->free_heaps->freelist = ((struct free_obj*)p)->next;
  if (gc->free_heaps->freelist == NULL) {
    unlink_free_heap_page(gc, gc->free_heaps);
  }

  gc->live++;
  gc_protect(mrb, gc, p);
  *(RVALUE *)p = RVALUE_zero;
  p->tt = ttype;
  p->c = cls;
  paint_partial_white(gc, p);
  return p;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

static double program_invoke_time = 0;

static double gc_time = 0;

static double gc_total_time = 0;

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define paint_partial_white(s, o) ((o)->color = (s)->current_white_part)

#define objects(p) ((RVALUE *)p->objects)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_incremental_gc(mrb_state *mrb)
{
  mrb_gc *gc = &mrb->gc;

  if (gc->disabled || gc->iterating) return;

  GC_INVOKE_TIME_REPORT("mrb_incremental_gc()");
  GC_TIME_START;

  if (is_minor_gc(gc)) {
    incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
  }
  else {
    incremental_gc_step(mrb, gc);
  }

  if (gc->state == MRB_GC_STATE_ROOT) {
    mrb_assert(gc->live >= gc->live_after_mark);
    gc->threshold = (gc->live_after_mark/100) * gc->interval_ratio;
    if (gc->threshold < GC_STEP_SIZE) {
      gc->threshold = GC_STEP_SIZE;
    }

    if (is_major_gc(gc)) {
      size_t threshold = gc->live_after_mark/100 * MAJOR_GC_INC_RATIO;

      gc->full = FALSE;
      if (threshold < MAJOR_GC_TOOMANY) {
        gc->majorgc_old_threshold = threshold;
      }
      else {
        /* too many objects allocated during incremental GC, */
        /* instead of increasing threshold, invoke full GC. */
        mrb_full_gc(mrb);
      }
    }
    else if (is_minor_gc(gc)) {
      if (gc->live > gc->majorgc_old_threshold) {
        clear_all_old(mrb, gc);
        gc->full = TRUE;
      }
    }
  }

  GC_TIME_STOP_AND_REPORT;
}

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

#define GC_INVOKE_TIME_REPORT(s)

#define GC_TIME_START

#define GC_TIME_STOP_AND_REPORT

#define GC_STEP_SIZE 1024

#define objects(p) ((RVALUE *)p->objects)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

#define MAJOR_GC_INC_RATIO 120

#define MAJOR_GC_TOOMANY 10000

#define is_major_gc(gc) (is_generational(gc) && (gc)->full)

#define is_minor_gc(gc) (is_generational(gc) && !(gc)->full)

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_garbage_collect(mrb_state *mrb)
{
  mrb_full_gc(mrb);
}


gc_protect(mrb_state *mrb, mrb_gc *gc, struct RBasic *p)
{
#ifdef MRB_GC_FIXED_ARENA
  if (gc->arena_idx >= MRB_GC_ARENA_SIZE) {
    /* arena overflow error */
    gc->arena_idx = MRB_GC_ARENA_SIZE - 4; /* force room in arena */
    mrb_exc_raise(mrb, mrb_obj_value(mrb->arena_err));
  }
#else
  if (gc->arena_idx >= gc->arena_capa) {
    /* extend arena */
    gc->arena_capa = (int)(gc->arena_capa * 3 / 2);
    gc->arena = (struct RBasic**)mrb_realloc(mrb, gc->arena, sizeof(struct RBasic*)*gc->arena_capa);
  }
#endif
  gc->arena[gc->arena_idx++] = p;
}


incremental_marking_phase(mrb_state *mrb, mrb_gc *gc, size_t limit)
{
  size_t tried_marks = 0;

  while (gc->gray_list && tried_marks < limit) {
    struct RBasic *obj = gc->gray_list;
    gc_mark_children(mrb, gc, obj);
    tried_marks += gc_gray_counts(mrb, gc, obj);
  }

  return tried_marks;
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define GC_INVOKE_TIME_REPORT(with) do {\
  fprintf(stderr, "%s\n", with);\
  fprintf(stderr, "gc_invoke: %19.3f\n", gettimeofday_time() - program_invoke_time);\
  fprintf(stderr, "is_generational: %d\n", is_generational(gc));\
  fprintf(stderr, "is_major_gc: %d\n", is_major_gc(gc));\
} while(0)

#define GC_TIME_START do {\
  gc_time = gettimeofday_time();\
} while(0)

#define GC_TIME_STOP_AND_REPORT do {\
  gc_time = gettimeofday_time() - gc_time;\
  gc_total_time += gc_time;\
  fprintf(stderr, "gc_state: %d\n", gc->state);\
  fprintf(stderr, "live: %zu\n", gc->live);\
  fprintf(stderr, "majorgc_old_threshold: %zu\n", gc->majorgc_old_threshold);\
  fprintf(stderr, "gc_threshold: %zu\n", gc->threshold);\
  fprintf(stderr, "gc_time: %30.20f\n", gc_time);\
  fprintf(stderr, "gc_total_time: %30.20f\n\n", gc_total_time);\
} while(0)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


mrb_realloc_simple(mrb_state *mrb, void *p,  size_t len)
{
  void *p2;

  p2 = (mrb->allocf)(mrb, p, len, mrb->allocf_ud);
  if (!p2 && len > 0 && mrb->gc.heaps) {
    mrb_full_gc(mrb);
    p2 = (mrb->allocf)(mrb, p, len, mrb->allocf_ud);
  }

  return p2;
}


mark_context_stack(mrb_state *mrb, struct mrb_context *c)
{
  size_t i;
  size_t e;
  mrb_value nil;

  if (c->stack == NULL) return;
  e = c->stack - c->stbase;
  if (c->ci) {
    e += ci_nregs(c->ci);
  }
  if (c->stbase + e > c->stend) e = c->stend - c->stbase;
  for (i=0; i<e; i++) {
    mrb_value v = c->stbase[i];

    if (!mrb_immediate_p(v)) {
      mrb_gc_mark(mrb, mrb_basic_ptr(v));
    }
  }
  e = c->stend - c->stbase;
  nil = mrb_nil_value();
  for (; i<e; i++) {
    c->stbase[i] = nil;
  }
}


mark_context(mrb_state *mrb, struct mrb_context *c)
{
  int i;
  mrb_callinfo *ci;

 start:
  if (c->status == MRB_FIBER_TERMINATED) return;

  /* mark VM stack */
  mark_context_stack(mrb, c);

  /* mark call stack */
  if (c->cibase) {
    for (ci = c->cibase; ci <= c->ci; ci++) {
      mrb_gc_mark(mrb, (struct RBasic*)ci->env);
      mrb_gc_mark(mrb, (struct RBasic*)ci->proc);
      mrb_gc_mark(mrb, (struct RBasic*)ci->target_class);
    }
  }
  /* mark ensure stack */
  for (i=0; i<c->eidx; i++) {
    mrb_gc_mark(mrb, (struct RBasic*)c->ensure[i]);
  }
  /* mark fibers */
  mrb_gc_mark(mrb, (struct RBasic*)c->fib);
  if (c->prev) {
    c = c->prev;
    goto start;
  }
}


clear_all_old(mrb_state *mrb, mrb_gc *gc)
{
  mrb_bool origin_mode = gc->generational;

  mrb_assert(is_generational(gc));
  if (is_major_gc(gc)) {
    /* finish the half baked GC */
    incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
  }

  /* Sweep the dead objects, then reset all the live objects
   * (including all the old objects, of course) to white. */
  gc->generational = FALSE;
  prepare_incremental_sweep(mrb, gc);
  incremental_gc_until(mrb, gc, MRB_GC_STATE_ROOT);
  gc->generational = origin_mode;

  /* The gray objects have already been painted as white */
  gc->atomic_gray_list = gc->gray_list = NULL;
}


add_heap(mrb_state *mrb, mrb_gc *gc)
{
  mrb_heap_page *page = (mrb_heap_page *)mrb_calloc(mrb, 1, sizeof(mrb_heap_page) + MRB_HEAP_PAGE_SIZE * sizeof(RVALUE));
  RVALUE *p, *e;
  struct RBasic *prev = NULL;

  for (p = objects(page), e=p+MRB_HEAP_PAGE_SIZE; p<e; p++) {
    p->as.free.tt = MRB_TT_FREE;
    p->as.free.next = prev;
    prev = &p->as.basic;
  }
  page->freelist = prev;

  link_heap_page(gc, page);
  link_free_heap_page(gc, page);
}


gc_interval_ratio_set(mrb_state *mrb, mrb_value obj)
{
  mrb_int ratio;

  mrb_get_args(mrb, "i", &ratio);
  mrb->gc.interval_ratio = (int)ratio;
  return mrb_nil_value();
}


gc_enable(mrb_state *mrb, mrb_value obj)
{
  mrb_bool old = mrb->gc.disabled;

  mrb->gc.disabled = FALSE;

  return mrb_bool_value(old);
}


gc_generational_mode_get(mrb_state *mrb, mrb_value self)
{
  return mrb_bool_value(mrb->gc.generational);
}


mrb_realloc(mrb_state *mrb, void *p, size_t len)
{
  void *p2;

  p2 = mrb_realloc_simple(mrb, p, len);
  if (len == 0) return p2;
  if (p2 == NULL) {
    mrb->gc.out_of_memory = TRUE;
    mrb_raise_nomemory(mrb);
  }
  else {
    mrb->gc.out_of_memory = FALSE;
  }

  return p2;
}


mrb_write_barrier(mrb_state *mrb, struct RBasic *obj)
{
  mrb_gc *gc = &mrb->gc;

  if (!is_black(obj)) return;

  mrb_assert(!is_dead(gc, obj));
  mrb_assert(is_generational(gc) || gc->state != MRB_GC_STATE_ROOT);
  paint_gray(obj);
  obj->gcnext = gc->atomic_gray_list;
  gc->atomic_gray_list = obj;
}


final_marking_phase(mrb_state *mrb, mrb_gc *gc)
{
  int i, e;

  /* mark arena */
  for (i=0,e=gc->arena_idx; i<e; i++) {
    mrb_gc_mark(mrb, gc->arena[i]);
  }
  mrb_gc_mark_gv(mrb);
  mark_context(mrb, mrb->c);
  mark_context(mrb, mrb->root_c);
  mrb_gc_mark(mrb, (struct RBasic*)mrb->exc);
  gc_mark_gray_list(mrb, gc);
  mrb_assert(gc->gray_list == NULL);
  gc->gray_list = gc->atomic_gray_list;
  gc->atomic_gray_list = NULL;
  gc_mark_gray_list(mrb, gc);
  mrb_assert(gc->gray_list == NULL);
}


gettimeofday_time(void)
{
  struct timeval tv;
  gettimeofday(&tv, NULL);
  return tv.tv_sec + tv.tv_usec * 1e-6;
}


gc_mark_children(mrb_state *mrb, mrb_gc *gc, struct RBasic *obj)
{
  mrb_assert(is_gray(obj));
  paint_black(obj);
  gc->gray_list = obj->gcnext;
  mrb_gc_mark(mrb, (struct RBasic*)obj->c);
  switch (obj->tt) {
  case MRB_TT_ICLASS:
    {
      struct RClass *c = (struct RClass*)obj;
      if (MRB_FLAG_TEST(c, MRB_FL_CLASS_IS_ORIGIN))
        mrb_gc_mark_mt(mrb, c);
      mrb_gc_mark(mrb, (struct RBasic*)((struct RClass*)obj)->super);
    }
    break;

  case MRB_TT_CLASS:
  case MRB_TT_MODULE:
  case MRB_TT_SCLASS:
    {
      struct RClass *c = (struct RClass*)obj;

      mrb_gc_mark_mt(mrb, c);
      mrb_gc_mark(mrb, (struct RBasic*)c->super);
    }
    /* fall through */

  case MRB_TT_OBJECT:
  case MRB_TT_DATA:
  case MRB_TT_EXCEPTION:
    mrb_gc_mark_iv(mrb, (struct RObject*)obj);
    break;

  case MRB_TT_PROC:
    {
      struct RProc *p = (struct RProc*)obj;

      mrb_gc_mark(mrb, (struct RBasic*)p->upper);
      mrb_gc_mark(mrb, (struct RBasic*)p->e.env);
    }
    break;

  case MRB_TT_ENV:
    {
      struct REnv *e = (struct REnv*)obj;
      mrb_int i, len;

      if (MRB_ENV_ONSTACK_P(e) && e->cxt && e->cxt->fib) {
        mrb_gc_mark(mrb, (struct RBasic*)e->cxt->fib);
      }
      len = MRB_ENV_LEN(e);
      for (i=0; i<len; i++) {
        mrb_gc_mark_value(mrb, e->stack[i]);
      }
    }
    break;

  case MRB_TT_FIBER:
    {
      struct mrb_context *c = ((struct RFiber*)obj)->cxt;

      if (c) mark_context(mrb, c);
    }
    break;

  case MRB_TT_ARRAY:
    {
      struct RArray *a = (struct RArray*)obj;
      size_t i, e;

      for (i=0,e=ARY_LEN(a); i<e; i++) {
        mrb_gc_mark_value(mrb, ARY_PTR(a)[i]);
      }
    }
    break;

  case MRB_TT_HASH:
    mrb_gc_mark_iv(mrb, (struct RObject*)obj);
    mrb_gc_mark_hash(mrb, (struct RHash*)obj);
    break;

  case MRB_TT_STRING:
    if (RSTR_FSHARED_P(obj)) {
      struct RString *s = (struct RString*)obj;
      mrb_gc_mark(mrb, (struct RBasic*)s->as.heap.aux.fshared);
    }
    break;

  case MRB_TT_RANGE:
    mrb_gc_mark_range(mrb, (struct RRange*)obj);
    break;

  default:
    break;
  }
}

struct free_obj {
  MRB_OBJECT_HEADER;
  struct RBasic *next;
}

typedef struct {
  union {
    struct free_obj free;
    struct RBasic basic;
    struct RObject object;
    struct RClass klass;
    struct RString string;
    struct RArray array;
    struct RHash hash;
    struct RRange range;
    struct RData data;
    struct RIStruct istruct;
    struct RProc proc;
    struct REnv env;
    struct RFiber fiber;
    struct RException exc;
    struct RBreak brk;
#ifdef MRB_WORD_BOXING
#ifndef MRB_WITHOUT_FLOAT
    struct RFloat floatv;
#endif
    struct RCptr cptr;
#endif
  } as;
} RVALUE;

#define paint_black(o) ((o)->color = GC_BLACK)

#define is_gray(o) ((o)->color == GC_GRAY)

mrb_noreturn void mrb_raise_nomemory(mrb_state *mrb);

static void obj_free(mrb_state *mrb, struct RBasic *obj, int end);

MRB_API struct

static mrb_value gc_test(mrb_state *, mrb_value);


  Status GetFirstDimensionSize(OpKernelContext* context, INDEX_TYPE* result) {
    const Tensor first_partition_tensor =
        context->input(kFirstPartitionInputIndex);
    if (row_partition_types_.empty()) {
      return errors::InvalidArgument("No row_partition_types given.");
    }
    const RowPartitionType first_partition_type = row_partition_types_[0];
    switch (first_partition_type) {
      case RowPartitionType::FIRST_DIM_SIZE:
        *result = first_partition_tensor.scalar<INDEX_TYPE>()();
        return Status::OK();
      case RowPartitionType::VALUE_ROWIDS:
        return errors::InvalidArgument(
            "Cannot handle VALUE_ROWIDS in first dimension.");
      case RowPartitionType::ROW_SPLITS:
        *result = first_partition_tensor.shape().dim_size(0) - 1;
        return Status::OK();
      default:
        return errors::InvalidArgument(
            "Cannot handle type ",
            RowPartitionTypeToString(first_partition_type));
    }
  }

Fix null ptr deref in tf.raw_ops.RaggedTensorToTensor

PiperOrigin-RevId: 384257511
Change-Id: I0484ad285039d132d6c41b284a7fcdd2b774a38e


  void Compute(OpKernelContext *ctx) override {
    // (0) validations
    const Tensor *a_indices, *b_indices, *a_values_t, *b_values_t, *a_shape,
        *b_shape, *thresh_t;

    OP_REQUIRES_OK(ctx, ctx->input("a_indices", &a_indices));
    OP_REQUIRES_OK(ctx, ctx->input("b_indices", &b_indices));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsMatrix(a_indices->shape()) &&
                    TensorShapeUtils::IsMatrix(b_indices->shape()),
                errors::InvalidArgument(
                    "Input indices should be matrices but received shapes: ",
                    a_indices->shape().DebugString(), " and ",
                    b_indices->shape().DebugString()));
    const int64 a_nnz = a_indices->dim_size(0);
    const int64 b_nnz = b_indices->dim_size(0);
    const int num_dims = a_indices->dim_size(1);
    OP_REQUIRES(ctx, b_indices->dim_size(1) == num_dims,
                errors::InvalidArgument(
                    "Input indices must have the same dimension, got ",
                    num_dims, " and ", b_indices->dim_size(1)));

    OP_REQUIRES_OK(ctx, ctx->input("a_values", &a_values_t));
    OP_REQUIRES_OK(ctx, ctx->input("b_values", &b_values_t));

    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_values_t->shape()) &&
                    TensorShapeUtils::IsVector(b_values_t->shape()),
                errors::InvalidArgument(
                    "Input values should be vectors but received shapes: ",
                    a_values_t->shape().DebugString(), " and ",
                    b_values_t->shape().DebugString()));
    auto a_values = ctx->input(1).vec<T>();
    auto b_values = ctx->input(4).vec<T>();
    OP_REQUIRES(
        ctx, a_values.size() == a_nnz && b_values.size() == b_nnz,
        errors::InvalidArgument("Expected ", a_nnz, " and ", b_nnz,
                                " non-empty input values, got ",
                                a_values.size(), " and ", b_values.size()));

    OP_REQUIRES_OK(ctx, ctx->input("a_shape", &a_shape));
    OP_REQUIRES_OK(ctx, ctx->input("b_shape", &b_shape));
    OP_REQUIRES(ctx,
                TensorShapeUtils::IsVector(a_shape->shape()) &&
                    TensorShapeUtils::IsVector(b_shape->shape()),
                errors::InvalidArgument(
                    "Input shapes should be a vector but received shapes ",
                    a_shape->shape().DebugString(), " and ",
                    b_shape->shape().DebugString()));
    OP_REQUIRES(
        ctx, a_shape->NumElements() == num_dims,
        errors::InvalidArgument("Second dimension of a_indices and length of "
                                "a_shape must match, got ",
                                num_dims, " and ", a_shape->NumElements()));
    OP_REQUIRES(ctx, num_dims > 0,
                errors::InvalidArgument("Tesors must not be empty"));
    OP_REQUIRES(
        ctx, a_shape->IsSameSize(*b_shape),
        errors::InvalidArgument(
            "Operands do not have the same ranks; got shapes: ",
            a_shape->SummarizeValue(10), " and ", b_shape->SummarizeValue(10)));
    const auto a_shape_flat = a_shape->flat<int64>();
    const auto b_shape_flat = b_shape->flat<int64>();
    for (int i = 0; i < a_shape->NumElements(); ++i) {
      OP_REQUIRES(ctx, a_shape_flat(i) == b_shape_flat(i),
                  errors::InvalidArgument(
                      "Operands' shapes do not match: got ", a_shape_flat(i),
                      " and ", b_shape_flat(i), " for dimension ", i));
    }

    OP_REQUIRES_OK(ctx, ctx->input("thresh", &thresh_t));
    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(thresh_t->shape()),
                errors::InvalidArgument(
                    "The magnitude threshold must be a scalar: got shape ",
                    thresh_t->shape().DebugString()));
    // std::abs() so that it works for complex{64,128} values as well
    const Treal thresh = thresh_t->scalar<Treal>()();

    // (1) do a pass over inputs, and append values and indices to vectors
    auto a_indices_mat = a_indices->matrix<int64>();
    auto b_indices_mat = b_indices->matrix<int64>();
    std::vector<std::pair<bool, int64>> entries_to_copy;  // from_a?, idx
    entries_to_copy.reserve(a_nnz + b_nnz);
    std::vector<T> out_values;

    // The input and output sparse tensors are assumed to be ordered along
    // increasing dimension number.
    int64 i = 0, j = 0;
    T s;
    while (i < a_nnz && j < b_nnz) {
      switch (sparse::DimComparator::cmp(a_indices_mat, b_indices_mat, i, j,
                                         num_dims)) {
        case -1:
          entries_to_copy.emplace_back(true, i);
          out_values.push_back(a_values(i));
          ++i;
          break;
        case 0:
          s = a_values(i) + b_values(j);
          if (thresh <= std::abs(s)) {
            entries_to_copy.emplace_back(true, i);
            out_values.push_back(s);
          }
          ++i;
          ++j;
          break;
        case 1:
          entries_to_copy.emplace_back(false, j);
          out_values.push_back(b_values(j));
          ++j;
          break;
      }
    }

#define HANDLE_LEFTOVERS(A_OR_B, IDX, IS_A)     \
  while (IDX < A_OR_B##_nnz) {                  \
    entries_to_copy.emplace_back(IS_A, IDX);    \
    out_values.push_back(A_OR_B##_values(IDX)); \
    ++IDX;                                      \
  }

    // at most one of these calls appends new values
    HANDLE_LEFTOVERS(a, i, true);
    HANDLE_LEFTOVERS(b, j, false);
#undef HANDLE_LEFTOVERS

    // (2) allocate and fill output tensors
    const int64 sum_nnz = out_values.size();
    Tensor *out_indices_t, *out_values_t;
    OP_REQUIRES_OK(ctx,
                   ctx->allocate_output(0, TensorShape({sum_nnz, num_dims}),
                                        &out_indices_t));
    OP_REQUIRES_OK(
        ctx, ctx->allocate_output(1, TensorShape({sum_nnz}), &out_values_t));
    auto out_indices_mat = out_indices_t->matrix<int64>();
    auto out_values_flat = out_values_t->vec<T>();

    for (i = 0; i < sum_nnz; ++i) {
      const bool from_a = entries_to_copy[i].first;
      const int64 idx = entries_to_copy[i].second;
      out_indices_mat.chip<0>(i) =
          from_a ? a_indices_mat.chip<0>(idx) : b_indices_mat.chip<0>(idx);
    }
    if (sum_nnz > 0) {
      std::copy_n(out_values.begin(), sum_nnz, &out_values_flat(0));
    }
    ctx->set_output(2, *a_shape);
  }

Validate that a and b are proper sparse tensors

PiperOrigin-RevId: 373248068
Change-Id: I0a2041a0747901b3f00387a6a3bce9bca6b0b3b1


void ldapsrv_notification_retry_setup(struct ldapsrv_service *service, bool force)
{
	struct ldapsrv_connection *conn = NULL;
	struct timeval retry;
	size_t num_pending = 0;
	size_t num_active = 0;

	if (force) {
		TALLOC_FREE(service->notification.retry);
		service->notification.generation += 1;
	}

	if (service->notification.retry != NULL) {
		return;
	}

	for (conn = service->connections; conn != NULL; conn = conn->next) {
		if (conn->pending_calls == NULL) {
			continue;
		}

		num_pending += 1;

		if (conn->pending_calls->notification.generation !=
		    service->notification.generation)
		{
			num_active += 1;
		}
	}

	if (num_pending == 0) {
		return;
	}

	if (num_active != 0) {
		retry = timeval_current_ofs(0, 100);
	} else {
		retry = timeval_current_ofs(5, 0);
	}

	service->notification.retry = tevent_wakeup_send(service,
							 service->task->event_ctx,
							 retry);
	if (service->notification.retry == NULL) {
		/* retry later */
		return;
	}

	tevent_req_set_callback(service->notification.retry,
				ldapsrv_notification_retry_done,
				service);
}

CVE-2021-3670 ldap_server: Ensure value of MaxQueryDuration is greater than zero

BUG: https://bugzilla.samba.org/show_bug.cgi?id=14694

Signed-off-by: Joseph Sutton <josephsutton@catalyst.net.nz>
Reviewed-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
(cherry picked from commit e1ab0c43629686d1d2c0b0b2bcdc90057a792049)

static void ldapsrv_terminate_connection_done(struct tevent_req *subreq);

static struct tevent_req *ldapsrv_process_call_send(TALLOC_CTX *mem_ctx,
						    struct tevent_context *ev,
						    struct tevent_queue *call_queue,
						    struct ldapsrv_call *call);

static NTSTATUS ldapsrv_process_call_recv(struct tevent_req *req);

static bool ldapsrv_call_read_next(struct ldapsrv_connection *conn);

static void ldapsrv_accept_tls_done(struct tevent_req *subreq);

static void ldapsrv_call_read_done(struct tevent_req *subreq);

static NTSTATUS ldapsrv_packet_check(
	void *private_data,
	DATA_BLOB blob,
	size_t *packet_size);

static void ldapsrv_call_process_done(struct tevent_req *subreq);

static int ldapsrv_check_packet_size(
	struct ldapsrv_connection *conn,
	size_t size);

static void ldapsrv_call_wait_done(struct tevent_req *subreq);

static void ldapsrv_call_writev_start(struct ldapsrv_call *call);

static void ldapsrv_call_writev_done(struct tevent_req *subreq);

static void ldapsrv_call_postprocess_done(struct tevent_req *subreq);

static void ldapsrv_notification_retry_done(struct tevent_req *subreq);

struct ldapsrv_process_call_state {
	struct ldapsrv_call *call;
}

static void ldapsrv_process_call_trigger(struct tevent_req *req,
					 void *private_data);

static void ldapsrv_disconnect_ticket_expired(struct tevent_req *subreq);

static const struct stream_server_ops ldap_stream_nonpriv_ops = {
	.name			= "ldap",
	.accept_connection	= ldapsrv_accept_nonpriv,
	.recv_handler		= ldapsrv_recv,
	.send_handler		= ldapsrv_send,
};

static const struct stream_server_ops ldap_stream_priv_ops = {
	.name			= "ldap",
	.accept_connection	= ldapsrv_accept_priv,
	.recv_handler		= ldapsrv_recv,
	.send_handler		= ldapsrv_send,
};


static void ldapsrv_post_fork(struct task_server *task, struct process_details *pd)
{
	struct ldapsrv_service *ldap_service =
		talloc_get_type_abort(task->private_data, struct ldapsrv_service);

	ldap_service->sam_ctx = samdb_connect(ldap_service,
					      ldap_service->task->event_ctx,
					      ldap_service->task->lp_ctx,
					      system_session(ldap_service->task->lp_ctx),
					      NULL,
					      0);
	if (ldap_service->sam_ctx == NULL) {
		task_server_terminate(task, "Cannot open system session LDB",
				      true);
		return;
	}
}


static int ldapsrv_call_destructor(struct ldapsrv_call *call)
{
	if (call->conn == NULL) {
		return 0;
	}

	DLIST_REMOVE(call->conn->pending_calls, call);

	call->conn = NULL;
	return 0;
}


static void ldapsrv_terminate_connection_done(struct tevent_req *subreq)
{
	struct ldapsrv_connection *conn =
		tevent_req_callback_data(subreq,
		struct ldapsrv_connection);
	int sys_errno;
	bool ok;

	tstream_disconnect_recv(subreq, &sys_errno);
	TALLOC_FREE(subreq);

	if (conn->sockets.active == conn->sockets.raw) {
		TALLOC_FREE(conn->sockets.tls);
		TALLOC_FREE(conn->sockets.sasl);
		TALLOC_FREE(conn->sockets.raw);
		stream_terminate_connection(conn->connection,
					    conn->limits.reason);
		return;
	}

	TALLOC_FREE(conn->sockets.tls);
	TALLOC_FREE(conn->sockets.sasl);
	conn->sockets.active = conn->sockets.raw;

	subreq = tstream_disconnect_send(conn,
					 conn->connection->event.ctx,
					 conn->sockets.active);
	if (subreq == NULL) {
		TALLOC_FREE(conn->sockets.raw);
		stream_terminate_connection(conn->connection,
					    conn->limits.reason);
		return;
	}
	ok = tevent_req_set_endtime(subreq,
				    conn->connection->event.ctx,
				    conn->limits.endtime);
	if (!ok) {
		TALLOC_FREE(conn->sockets.raw);
		stream_terminate_connection(conn->connection,
					    conn->limits.reason);
		return;
	}
	tevent_req_set_callback(subreq, ldapsrv_terminate_connection_done, conn);
}


static void ldapsrv_accept(struct stream_connection *c,
			   struct auth_session_info *session_info,
			   bool is_privileged)
{
	struct ldapsrv_service *ldapsrv_service =
		talloc_get_type(c->private_data, struct ldapsrv_service);
	struct ldapsrv_connection *conn;
	struct cli_credentials *server_credentials;
	struct socket_address *socket_address;
	NTSTATUS status;
	int port;
	int ret;
	struct tevent_req *subreq;
	struct timeval endtime;
	char *errstring = NULL;

	conn = talloc_zero(c, struct ldapsrv_connection);
	if (!conn) {
		stream_terminate_connection(c, "ldapsrv_accept: out of memory");
		return;
	}
	conn->is_privileged = is_privileged;

	conn->sockets.send_queue = tevent_queue_create(conn, "ldapsev send queue");
	if (conn->sockets.send_queue == NULL) {
		stream_terminate_connection(c,
					    "ldapsrv_accept: tevent_queue_create failed");
		return;
	}

	TALLOC_FREE(c->event.fde);

	ret = tstream_bsd_existing_socket(conn,
					  socket_get_fd(c->socket),
					  &conn->sockets.raw);
	if (ret == -1) {
		stream_terminate_connection(c,
					    "ldapsrv_accept: out of memory");
		return;
	}
	socket_set_flags(c->socket, SOCKET_FLAG_NOCLOSE);

	conn->connection  = c;
	conn->service     = ldapsrv_service;
	conn->lp_ctx      = ldapsrv_service->task->lp_ctx;

	c->private_data   = conn;

	socket_address = socket_get_my_addr(c->socket, conn);
	if (!socket_address) {
		ldapsrv_terminate_connection(conn, "ldapsrv_accept: failed to obtain local socket address!");
		return;
	}
	port = socket_address->port;
	talloc_free(socket_address);
	if (port == 3268 || port == 3269) /* Global catalog */ {
		conn->global_catalog = true;
	}

	server_credentials = cli_credentials_init(conn);
	if (!server_credentials) {
		stream_terminate_connection(c, "Failed to init server credentials\n");
		return;
	}

	cli_credentials_set_conf(server_credentials, conn->lp_ctx);
	status = cli_credentials_set_machine_account(server_credentials, conn->lp_ctx);
	if (!NT_STATUS_IS_OK(status)) {
		stream_terminate_connection(c, talloc_asprintf(conn, "Failed to obtain server credentials, perhaps a standalone server?: %s\n", nt_errstr(status)));
		return;
	}
	conn->server_credentials = server_credentials;

	conn->session_info = session_info;

	conn->sockets.active = conn->sockets.raw;

	if (conn->is_privileged) {
		conn->require_strong_auth = LDAP_SERVER_REQUIRE_STRONG_AUTH_NO;
	} else {
		conn->require_strong_auth = lpcfg_ldap_server_require_strong_auth(conn->lp_ctx);
	}

	ret = ldapsrv_backend_Init(conn, &errstring);
	if (ret != LDB_SUCCESS) {
		char *reason = talloc_asprintf(conn,
					       "LDB backend for LDAP Init "
					       "failed: %s: %s",
					       errstring, ldb_strerror(ret));
		ldapsrv_terminate_connection(conn, reason);
		return;
	}

	/* load limits from the conf partition */
	ldapsrv_load_limits(conn); /* should we fail on error ? */

	/* register the server */
	irpc_add_name(c->msg_ctx, "ldap_server");

	DLIST_ADD_END(ldapsrv_service->connections, conn);

	if (port != 636 && port != 3269) {
		ldapsrv_call_read_next(conn);
		return;
	}

	endtime = timeval_current_ofs(conn->limits.conn_idle_time, 0);

	subreq = tstream_tls_accept_send(conn,
					 conn->connection->event.ctx,
					 conn->sockets.raw,
					 conn->service->tls_params);
	if (subreq == NULL) {
		ldapsrv_terminate_connection(conn, "ldapsrv_accept: "
				"no memory for tstream_tls_accept_send");
		return;
	}
	tevent_req_set_endtime(subreq,
			       conn->connection->event.ctx,
			       endtime);
	tevent_req_set_callback(subreq, ldapsrv_accept_tls_done, conn);
}


static void ldapsrv_notification_retry_done(struct tevent_req *subreq)
{
	struct ldapsrv_service *service =
		tevent_req_callback_data(subreq,
		struct ldapsrv_service);
	struct ldapsrv_connection *conn = NULL;
	struct ldapsrv_connection *conn_next = NULL;
	bool ok;

	service->notification.retry = NULL;

	ok = tevent_wakeup_recv(subreq);
	TALLOC_FREE(subreq);
	if (!ok) {
		/* ignore */
	}

	for (conn = service->connections; conn != NULL; conn = conn_next) {
		struct ldapsrv_call *call = conn->pending_calls;

		conn_next = conn->next;

		if (conn->pending_calls == NULL) {
			continue;
		}

		if (conn->active_call != NULL) {
			continue;
		}

		DLIST_DEMOTE(conn->pending_calls, call);
		call->notification.generation =
				service->notification.generation;

		/* queue the call in the global queue */
		subreq = ldapsrv_process_call_send(call,
						   conn->connection->event.ctx,
						   conn->service->call_queue,
						   call);
		if (subreq == NULL) {
			ldapsrv_terminate_connection(conn,
					"ldapsrv_process_call_send failed");
			continue;
		}
		tevent_req_set_callback(subreq, ldapsrv_call_process_done, call);
		conn->active_call = subreq;
	}

	ldapsrv_notification_retry_setup(service, false);
}


static NTSTATUS add_socket(struct task_server *task,
			   struct loadparm_context *lp_ctx,
			   const struct model_ops *model_ops,
			   const char *address, struct ldapsrv_service *ldap_service)
{
	uint16_t port = 389;
	NTSTATUS status;
	struct ldb_context *ldb;

	status = stream_setup_socket(task, task->event_ctx, lp_ctx,
				     model_ops, &ldap_stream_nonpriv_ops,
				     "ip", address, &port,
				     lpcfg_socket_options(lp_ctx),
				     ldap_service, task->process_context);
	if (!NT_STATUS_IS_OK(status)) {
		DBG_ERR("ldapsrv failed to bind to %s:%u - %s\n",
			address, port, nt_errstr(status));
		return status;
	}

	if (tstream_tls_params_enabled(ldap_service->tls_params)) {
		/* add ldaps server */
		port = 636;
		status = stream_setup_socket(task, task->event_ctx, lp_ctx,
					     model_ops,
					     &ldap_stream_nonpriv_ops,
					     "ip", address, &port,
					     lpcfg_socket_options(lp_ctx),
					     ldap_service,
					     task->process_context);
		if (!NT_STATUS_IS_OK(status)) {
			DBG_ERR("ldapsrv failed to bind to %s:%u - %s\n",
				address, port, nt_errstr(status));
			return status;
		}
	}

	/* Load LDAP database, but only to read our settings */
	ldb = samdb_connect(ldap_service,
			    ldap_service->task->event_ctx,
			    lp_ctx,
			    system_session(lp_ctx),
			    NULL,
			    0);
	if (!ldb) {
		return NT_STATUS_INTERNAL_DB_CORRUPTION;
	}

	if (samdb_is_gc(ldb)) {
		port = 3268;
		status = stream_setup_socket(task, task->event_ctx, lp_ctx,
					     model_ops,
					     &ldap_stream_nonpriv_ops,
					     "ip", address, &port,
					     lpcfg_socket_options(lp_ctx),
					     ldap_service,
					     task->process_context);
		if (!NT_STATUS_IS_OK(status)) {
			DBG_ERR("ldapsrv failed to bind to %s:%u - %s\n",
				address, port, nt_errstr(status));
			return status;
		}
		if (tstream_tls_params_enabled(ldap_service->tls_params)) {
			/* add ldaps server for the global catalog */
			port = 3269;
			status = stream_setup_socket(task, task->event_ctx, lp_ctx,
						     model_ops,
						     &ldap_stream_nonpriv_ops,
						     "ip", address, &port,
						     lpcfg_socket_options(lp_ctx),
						     ldap_service,
						     task->process_context);
			if (!NT_STATUS_IS_OK(status)) {
				DBG_ERR("ldapsrv failed to bind to %s:%u - %s\n",
					address, port, nt_errstr(status));
				return status;
			}
		}
	}

	/* And once we are bound, free the temporary ldb, it will
	 * connect again on each incoming LDAP connection */
	talloc_unlink(ldap_service, ldb);

	return NT_STATUS_OK;
}


static void ldapsrv_call_read_done(struct tevent_req *subreq)
{
	struct ldapsrv_connection *conn =
		tevent_req_callback_data(subreq,
		struct ldapsrv_connection);
	NTSTATUS status;
	struct ldapsrv_call *call;
	struct asn1_data *asn1;
	DATA_BLOB blob;
	int ret = LDAP_SUCCESS;
	struct ldap_request_limits limits = {0};

	conn->sockets.read_req = NULL;

	call = talloc_zero(conn, struct ldapsrv_call);
	if (!call) {
		ldapsrv_terminate_connection(conn, "no memory");
		return;
	}
	talloc_set_destructor(call, ldapsrv_call_destructor);

	call->conn = conn;

	status = tstream_read_pdu_blob_recv(subreq,
					    call,
					    &blob);
	TALLOC_FREE(subreq);
	if (!NT_STATUS_IS_OK(status)) {
		const char *reason;

		reason = talloc_asprintf(call, "ldapsrv_call_loop: "
					 "tstream_read_pdu_blob_recv() - %s",
					 nt_errstr(status));
		if (!reason) {
			reason = nt_errstr(status);
		}

		ldapsrv_terminate_connection(conn, reason);
		return;
	}

	ret = ldapsrv_check_packet_size(conn, blob.length);
	if (ret != LDAP_SUCCESS) {
		ldapsrv_terminate_connection(
			conn,
			"Request packet too large");
		return;
	}

	asn1 = asn1_init(call, ASN1_MAX_TREE_DEPTH);
	if (asn1 == NULL) {
		ldapsrv_terminate_connection(conn, "no memory");
		return;
	}

	call->request = talloc(call, struct ldap_message);
	if (call->request == NULL) {
		ldapsrv_terminate_connection(conn, "no memory");
		return;
	}

	asn1_load_nocopy(asn1, blob.data, blob.length);

	limits.max_search_size =
		lpcfg_ldap_max_search_request_size(conn->lp_ctx);
	status = ldap_decode(
		asn1,
		&limits,
		samba_ldap_control_handlers(),
		call->request);
	if (!NT_STATUS_IS_OK(status)) {
		ldapsrv_terminate_connection(conn, nt_errstr(status));
		return;
	}

	data_blob_free(&blob);
	TALLOC_FREE(asn1);

	/* queue the call in the global queue */
	subreq = ldapsrv_process_call_send(call,
					   conn->connection->event.ctx,
					   conn->service->call_queue,
					   call);
	if (subreq == NULL) {
		ldapsrv_terminate_connection(conn, "ldapsrv_process_call_send failed");
		return;
	}
	tevent_req_set_callback(subreq, ldapsrv_call_process_done, call);
	conn->active_call = subreq;
}


static int ldapsrv_check_packet_size(
	struct ldapsrv_connection *conn,
	size_t size)
{
	bool is_anonymous = false;
	size_t max_size = 0;

	max_size = lpcfg_ldap_max_anonymous_request_size(conn->lp_ctx);
	if (size <= max_size) {
		return LDAP_SUCCESS;
	}

	/*
	 * Request is larger than the maximum unauthenticated request size.
	 * As this code is called frequently we avoid calling
	 * security_token_is_anonymous if possible
	 */
	if (conn->session_info != NULL &&
		conn->session_info->security_token != NULL) {
		is_anonymous = security_token_is_anonymous(
			conn->session_info->security_token);
	}

	if (is_anonymous) {
		DBG_WARNING(
			"LDAP request size (%zu) exceeds (%zu)\n",
			size,
			max_size);
		return LDAP_UNWILLING_TO_PERFORM;
	}

	max_size = lpcfg_ldap_max_authenticated_request_size(conn->lp_ctx);
	if (size > max_size) {
		DBG_WARNING(
			"LDAP request size (%zu) exceeds (%zu)\n",
			size,
			max_size);
		return LDAP_UNWILLING_TO_PERFORM;
	}
	return LDAP_SUCCESS;

}


static void ldapsrv_process_call_trigger(struct tevent_req *req,
					 void *private_data)
{
	struct ldapsrv_process_call_state *state =
		tevent_req_data(req,
		struct ldapsrv_process_call_state);
	struct ldapsrv_connection *conn = state->call->conn;
	NTSTATUS status;

	if (conn->deferred_expire_disconnect != NULL) {
		/*
		 * Just drop this on the floor
		 */
		tevent_req_done(req);
		return;
	}

	/* make the call */
	status = ldapsrv_do_call(state->call);

	if (NT_STATUS_EQUAL(status, NT_STATUS_NETWORK_SESSION_EXPIRED)) {
		/*
		 * For testing purposes, defer the TCP disconnect
		 * after having sent the msgid 0
		 * 1.3.6.1.4.1.1466.20036 exop response. LDAP clients
		 * should not wait for the TCP connection to close but
		 * handle this packet equivalent to a TCP
		 * disconnect. This delay enables testing both cases
		 * in LDAP client libraries.
		 */

		int defer_msec = lpcfg_parm_int(
			conn->lp_ctx,
			NULL,
			"ldap_server",
			"delay_expire_disconnect",
			0);

		conn->deferred_expire_disconnect = tevent_wakeup_send(
			conn,
			conn->connection->event.ctx,
			timeval_current_ofs_msec(defer_msec));
		if (tevent_req_nomem(conn->deferred_expire_disconnect, req)) {
			return;
		}
		tevent_req_set_callback(
			conn->deferred_expire_disconnect,
			ldapsrv_disconnect_ticket_expired,
			conn);

		tevent_req_done(req);
		return;
	}

	if (!NT_STATUS_IS_OK(status)) {
		tevent_req_nterror(req, status);
		return;
	}

	tevent_req_done(req);
}


NTSTATUS server_service_ldap_init(TALLOC_CTX *ctx)
{
	static const struct service_details details = {
		.inhibit_fork_on_accept = false,
		.inhibit_pre_fork = false,
		.task_init = ldapsrv_task_init,
		.post_fork = ldapsrv_post_fork,
	};
	return register_server_service(ctx, "ldap", &details);
}


static void ldapsrv_terminate_connection(struct ldapsrv_connection *conn,
					 const char *reason)
{
	struct tevent_req *subreq;

	if (conn->limits.reason) {
		return;
	}

	DLIST_REMOVE(conn->service->connections, conn);

	conn->limits.endtime = timeval_current_ofs(0, 500);

	tevent_queue_stop(conn->sockets.send_queue);
	TALLOC_FREE(conn->sockets.read_req);
	TALLOC_FREE(conn->deferred_expire_disconnect);
	if (conn->active_call) {
		tevent_req_cancel(conn->active_call);
		conn->active_call = NULL;
	}

	conn->limits.reason = talloc_strdup(conn, reason);
	if (conn->limits.reason == NULL) {
		TALLOC_FREE(conn->sockets.tls);
		TALLOC_FREE(conn->sockets.sasl);
		TALLOC_FREE(conn->sockets.raw);
		stream_terminate_connection(conn->connection, reason);
		return;
	}

	subreq = tstream_disconnect_send(conn,
					 conn->connection->event.ctx,
					 conn->sockets.active);
	if (subreq == NULL) {
		TALLOC_FREE(conn->sockets.tls);
		TALLOC_FREE(conn->sockets.sasl);
		TALLOC_FREE(conn->sockets.raw);
		stream_terminate_connection(conn->connection, reason);
		return;
	}
	tevent_req_set_endtime(subreq,
			       conn->connection->event.ctx,
			       conn->limits.endtime);
	tevent_req_set_callback(subreq, ldapsrv_terminate_connection_done, conn);
}


static NTSTATUS ldapsrv_process_call_recv(struct tevent_req *req)
{
	NTSTATUS status;

	if (tevent_req_is_nterror(req, &status)) {
		tevent_req_received(req);
		return status;
	}

	tevent_req_received(req);
	return NT_STATUS_OK;
}


static void ldapsrv_call_wait_done(struct tevent_req *subreq)
{
	struct ldapsrv_call *call =
		tevent_req_callback_data(subreq,
		struct ldapsrv_call);
	struct ldapsrv_connection *conn = call->conn;
	NTSTATUS status;

	conn->active_call = NULL;

	status = call->wait_recv(subreq);
	TALLOC_FREE(subreq);
	if (!NT_STATUS_IS_OK(status)) {
		const char *reason;

		reason = talloc_asprintf(call, "ldapsrv_call_wait_done: "
					 "call->wait_recv() - %s",
					 nt_errstr(status));
		if (reason == NULL) {
			reason = nt_errstr(status);
		}

		ldapsrv_terminate_connection(conn, reason);
		return;
	}

	ldapsrv_call_writev_start(call);
}


static void ldapsrv_call_postprocess_done(struct tevent_req *subreq)
{
	struct ldapsrv_call *call =
		tevent_req_callback_data(subreq,
		struct ldapsrv_call);
	struct ldapsrv_connection *conn = call->conn;
	NTSTATUS status;

	status = call->postprocess_recv(subreq);
	TALLOC_FREE(subreq);
	if (!NT_STATUS_IS_OK(status)) {
		const char *reason;

		reason = talloc_asprintf(call, "ldapsrv_call_postprocess_done: "
					 "call->postprocess_recv() - %s",
					 nt_errstr(status));
		if (reason == NULL) {
			reason = nt_errstr(status);
		}

		ldapsrv_terminate_connection(conn, reason);
		return;
	}

	TALLOC_FREE(call);

	ldapsrv_call_read_next(conn);
}


static struct tevent_req *ldapsrv_process_call_send(TALLOC_CTX *mem_ctx,
						    struct tevent_context *ev,
						    struct tevent_queue *call_queue,
						    struct ldapsrv_call *call)
{
	struct tevent_req *req;
	struct ldapsrv_process_call_state *state;
	bool ok;

	req = tevent_req_create(mem_ctx, &state,
				struct ldapsrv_process_call_state);
	if (req == NULL) {
		return req;
	}

	state->call = call;

	ok = tevent_queue_add(call_queue, ev, req,
			      ldapsrv_process_call_trigger, NULL);
	if (!ok) {
		tevent_req_oom(req);
		return tevent_req_post(req, ev);
	}

	return req;
}


static void ldapsrv_disconnect_ticket_expired(struct tevent_req *subreq)
{
	struct ldapsrv_connection *conn = tevent_req_callback_data(
		subreq, struct ldapsrv_connection);
	bool ok;

	ok = tevent_wakeup_recv(subreq);
	TALLOC_FREE(subreq);
	if (!ok) {
		DBG_WARNING("tevent_wakeup_recv failed\n");
	}
	conn->deferred_expire_disconnect = NULL;
	ldapsrv_terminate_connection(conn, "network session expired");
}