static bool demux_mf_read_packet(struct demuxer *demuxer,
                                 struct demux_packet **pkt)
{
    mf_t *mf = demuxer->priv;
    if (mf->curr_frame >= mf->nr_of_files)
        return false;
    bool ok = false;

    struct stream *entry_stream = NULL;
    if (mf->streams)
        entry_stream = mf->streams[mf->curr_frame];
    struct stream *stream = entry_stream;
    if (!stream) {
        char *filename = mf->names[mf->curr_frame];
        if (filename) {
            stream = stream_create(filename, demuxer->stream_origin | STREAM_READ,
                                   demuxer->cancel, demuxer->global);
        }
    }

    if (stream) {
        stream_seek(stream, 0);
        bstr data = stream_read_complete(stream, NULL, MF_MAX_FILE_SIZE);
        if (data.len) {
            demux_packet_t *dp = new_demux_packet(data.len);
            if (dp) {
                memcpy(dp->buffer, data.start, data.len);
                dp->pts = mf->curr_frame / mf->sh->codec->fps;
                dp->keyframe = true;
                dp->stream = mf->sh->index;
                *pkt = dp;
                ok = true;
            }
        }
        talloc_free(data.start);
    }

    if (stream && stream != entry_stream)
        free_stream(stream);

    mf->curr_frame++;

    if (!ok)
        MP_ERR(demuxer, "error reading image file\n");

    return true;
}

#define MF_MAX_FILE_SIZE (1024 * 1024 * 256)

typedef struct mf {
    struct mp_log *log;
    struct sh_stream *sh;
    int curr_frame;
    int nr_of_files;
    char **names;
    // optional
    struct stream **streams;
} mf_t;

static const struct {
    const char *type;
    const char *codec;
} type2format[] = {
    { "bmp",            "bmp" },
    { "dpx",            "dpx" },
    { "j2c",            "jpeg2000" },
    { "j2k",            "jpeg2000" },
    { "jp2",            "jpeg2000" },
    { "jpc",            "jpeg2000" },
    { "jpeg",           "mjpeg" },
    { "jpg",            "mjpeg" },
    { "jps",            "mjpeg" },
    { "jls",            "ljpeg" },
    { "thm",            "mjpeg" },
    { "db",             "mjpeg" },
    { "pcd",            "photocd" },
    { "pfm",            "pfm" },
    { "pcx",            "pcx" },
    { "png",            "png" },
    { "pns",            "png" },
    { "ptx",            "ptx" },
    { "tga",            "targa" },
    { "tif",            "tiff" },
    { "tiff",           "tiff" },
    { "sgi",            "sgi" },
    { "sun",            "sunrast" },
    { "ras",            "sunrast" },
    { "rs",             "sunrast" },
    { "ra",             "sunrast" },
    { "im1",            "sunrast" },
    { "im8",            "sunrast" },
    { "im24",           "sunrast" },
    { "im32",           "sunrast" },
    { "sunras",         "sunrast" },
    { "xbm",            "xbm" },
    { "pam",            "pam" },
    { "pbm",            "pbm" },
    { "pgm",            "pgm" },
    { "pgmyuv",         "pgmyuv" },
    { "ppm",            "ppm" },
    { "pnm",            "ppm" },
    { "gif",            "gif" }, // usually handled by demux_lavf
    { "pix",            "brender_pix" },
    { "exr",            "exr" },
    { "pic",            "pictor" },
    { "xface",          "xface" },
    { "xwd",            "xwd" },
    {0}
};

const demuxer_desc_t demuxer_desc_mf = {
    .name = "mf",
    .desc = "image files (mf)",
    .read_packet = demux_mf_read_packet,
    .open = demux_open_mf,
    .close = demux_close_mf,
    .seek = demux_seek_mf,
};


static void mf_add(mf_t *mf, const char *fname)
{
    char *entry = talloc_strdup(mf, fname);
    MP_TARRAY_APPEND(mf, mf->names, mf->nr_of_files, entry);
}

demux_mf: improve format string processing

Before this commit, the user could specify a printf format string
which wasn't verified, and could result in:
- Undefined behavior due to missing or non-matching arguments.
- Buffer overflow due to untested result length.

The offending code was added at commit 103a9609 (2002, mplayer svn):
git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@4566 b3059339-0415-0410-9bf9-f77b7e298cf2

It moved around but was not modified meaningfully until now.

Now we reject all conversion specifiers at the format except %%
and a simple subset of the valid specifiers. Also, we now use
snprintf to avoid buffer overflow.

The format string is provided by the user as part of mf:// URI.

Report and initial patch by Stefan Schiller.
Patch reviewed by @jeeb, @sfan5, Stefan Schiller.

typedef struct mf {
    struct mp_log *log;
    struct sh_stream *sh;
    int curr_frame;
    int nr_of_files;
    char **names;
    // optional
    struct stream **streams;
} mf_t;

static const struct {
    const char *type;
    const char *codec;
} type2format[] = {
    { "bmp",            "bmp" },
    { "dpx",            "dpx" },
    { "j2c",            "jpeg2000" },
    { "j2k",            "jpeg2000" },
    { "jp2",            "jpeg2000" },
    { "jpc",            "jpeg2000" },
    { "jpeg",           "mjpeg" },
    { "jpg",            "mjpeg" },
    { "jps",            "mjpeg" },
    { "jls",            "ljpeg" },
    { "thm",            "mjpeg" },
    { "db",             "mjpeg" },
    { "pcd",            "photocd" },
    { "pfm",            "pfm" },
    { "pcx",            "pcx" },
    { "png",            "png" },
    { "pns",            "png" },
    { "ptx",            "ptx" },
    { "tga",            "targa" },
    { "tif",            "tiff" },
    { "tiff",           "tiff" },
    { "sgi",            "sgi" },
    { "sun",            "sunrast" },
    { "ras",            "sunrast" },
    { "rs",             "sunrast" },
    { "ra",             "sunrast" },
    { "im1",            "sunrast" },
    { "im8",            "sunrast" },
    { "im24",           "sunrast" },
    { "im32",           "sunrast" },
    { "sunras",         "sunrast" },
    { "xbm",            "xbm" },
    { "pam",            "pam" },
    { "pbm",            "pbm" },
    { "pgm",            "pgm" },
    { "pgmyuv",         "pgmyuv" },
    { "ppm",            "ppm" },
    { "pnm",            "ppm" },
    { "gif",            "gif" }, // usually handled by demux_lavf
    { "pix",            "brender_pix" },
    { "exr",            "exr" },
    { "pic",            "pictor" },
    { "xface",          "xface" },
    { "xwd",            "xwd" },
    {0}
};

const demuxer_desc_t demuxer_desc_mf = {
    .name = "mf",
    .desc = "image files (mf)",
    .read_packet = demux_mf_read_packet,
    .open = demux_open_mf,
    .close = demux_close_mf,
    .seek = demux_seek_mf,
};


static mf_t *open_mf_pattern(void *talloc_ctx, struct demuxer *d, char *filename)
{
    struct mp_log *log = d->log;
    int error_count = 0;
    int count = 0;

    mf_t *mf = talloc_zero(talloc_ctx, mf_t);
    mf->log = log;

    if (filename[0] == '@') {
        struct stream *s = stream_create(filename + 1,
                            d->stream_origin | STREAM_READ, d->cancel, d->global);
        if (s) {
            while (1) {
                char buf[512];
                int len = stream_read_peek(s, buf, sizeof(buf));
                if (!len)
                    break;
                bstr data = (bstr){buf, len};
                int pos = bstrchr(data, '\n');
                data = bstr_splice(data, 0, pos < 0 ? data.len : pos + 1);
                bstr fname = bstr_strip(data);
                if (fname.len) {
                    if (bstrchr(fname, '\0') >= 0) {
                        mp_err(log, "invalid filename\n");
                        break;
                    }
                    char *entry = bstrto0(mf, fname);
                    if (!mp_path_exists(entry)) {
                        mp_verbose(log, "file not found: '%s'\n", entry);
                    } else {
                        MP_TARRAY_APPEND(mf, mf->names, mf->nr_of_files, entry);
                    }
                }
                stream_seek_skip(s, stream_tell(s) + data.len);
            }
            free_stream(s);

            mp_info(log, "number of files: %d\n", mf->nr_of_files);
            goto exit_mf;
        }
        mp_info(log, "%s is not indirect filelist\n", filename + 1);
    }

    if (strchr(filename, ',')) {
        mp_info(log, "filelist: %s\n", filename);
        bstr bfilename = bstr0(filename);

        while (bfilename.len) {
            bstr bfname;
            bstr_split_tok(bfilename, ",", &bfname, &bfilename);
            char *fname2 = bstrdup0(mf, bfname);

            if (!mp_path_exists(fname2))
                mp_verbose(log, "file not found: '%s'\n", fname2);
            else {
                mf_add(mf, fname2);
            }
            talloc_free(fname2);
        }
        mp_info(log, "number of files: %d\n", mf->nr_of_files);

        goto exit_mf;
    }

    size_t fname_avail = strlen(filename) + 32;
    char *fname = talloc_size(mf, fname_avail);

#if HAVE_GLOB
    if (!strchr(filename, '%')) {
        strcpy(fname, filename);
        if (!strchr(filename, '*'))
            strcat(fname, "*");

        mp_info(log, "search expr: %s\n", fname);

        glob_t gg;
        if (glob(fname, 0, NULL, &gg)) {
            talloc_free(mf);
            return NULL;
        }

        for (int i = 0; i < gg.gl_pathc; i++) {
            if (mp_path_isdir(gg.gl_pathv[i]))
                continue;
            mf_add(mf, gg.gl_pathv[i]);
        }
        mp_info(log, "number of files: %d\n", mf->nr_of_files);
        globfree(&gg);
        goto exit_mf;
    }
#endif

    // We're using arbitrary user input as printf format with 1 int argument.
    // Any format which uses exactly 1 int argument would be valid, but for
    // simplicity we reject all conversion specifiers except %% and simple
    // integer specifier: %[.][NUM]d where NUM is 1-3 digits (%.d is valid)
    const char *f = filename;
    int MAXDIGS = 3, nspec = 0, bad_spec = 0, c;

    while (nspec < 2 && (c = *f++)) {
        if (c != '%')
            continue;
        if (*f != '%') {
            nspec++;  // conversion specifier which isn't %%
            if (*f == '.')
                f++;
            for (int ndig = 0; mp_isdigit(*f) && ndig < MAXDIGS; ndig++, f++)
                /* no-op */;
            if (*f != 'd') {
                bad_spec++;  // not int, or beyond our validation capacity
                break;
            }
        }
        // *f is '%' or 'd'
        f++;
    }

    // nspec==0 (zero specifiers) is rejected because fname wouldn't advance.
    if (bad_spec || nspec != 1) {
        mp_err(log, "unsupported expr format: '%s'\n", filename);
        goto exit_mf;
    }

    mp_info(log, "search expr: %s\n", filename);

    while (error_count < 5) {
        if (snprintf(fname, fname_avail, filename, count++) >= fname_avail) {
            mp_err(log, "format result too long: '%s'\n", filename);
            goto exit_mf;
        }
        if (!mp_path_exists(fname)) {
            error_count++;
            mp_verbose(log, "file not found: '%s'\n", fname);
        } else {
            mf_add(mf, fname);
        }
    }

    mp_info(log, "number of files: %d\n", mf->nr_of_files);

exit_mf:
    return mf;
}


static int demux_open_mf(demuxer_t *demuxer, enum demux_check check)
{
    mf_t *mf;

    if (strncmp(demuxer->stream->url, "mf://", 5) == 0 &&
        demuxer->stream->info && strcmp(demuxer->stream->info->name, "mf") == 0)
    {
        mf = open_mf_pattern(demuxer, demuxer, demuxer->stream->url + 5);
    } else {
        mf = open_mf_single(demuxer, demuxer->log, demuxer->stream->url);
        int bog = 0;
        MP_TARRAY_APPEND(mf, mf->streams, bog, demuxer->stream);
    }

    if (!mf || mf->nr_of_files < 1)
        goto error;

    double mf_fps;
    char *mf_type;
    mp_read_option_raw(demuxer->global, "mf-fps", &m_option_type_double, &mf_fps);
    mp_read_option_raw(demuxer->global, "mf-type", &m_option_type_string, &mf_type);

    const char *codec = mp_map_mimetype_to_video_codec(demuxer->stream->mime_type);
    if (!codec || (mf_type && mf_type[0]))
        codec = probe_format(mf, mf_type, check);
    talloc_free(mf_type);
    if (!codec)
        goto error;

    mf->curr_frame = 0;

    // create a new video stream header
    struct sh_stream *sh = demux_alloc_sh_stream(STREAM_VIDEO);
    struct mp_codec_params *c = sh->codec;

    c->codec = codec;
    c->disp_w = 0;
    c->disp_h = 0;
    c->fps = mf_fps;
    c->reliable_fps = true;

    demux_add_sh_stream(demuxer, sh);

    mf->sh = sh;
    demuxer->priv = (void *)mf;
    demuxer->seekable = true;
    demuxer->duration = mf->nr_of_files / mf->sh->codec->fps;

    return 0;

error:
    return -1;
}


static void demux_close_mf(demuxer_t *demuxer)
{
}

static const struct {
    const char *type;
    const char *codec;
} type2format[] = {
    { "bmp",            "bmp" },
    { "dpx",            "dpx" },
    { "j2c",            "jpeg2000" },
    { "j2k",            "jpeg2000" },
    { "jp2",            "jpeg2000" },
    { "jpc",            "jpeg2000" },
    { "jpeg",           "mjpeg" },
    { "jpg",            "mjpeg" },
    { "jps",            "mjpeg" },
    { "jls",            "ljpeg" },
    { "thm",            "mjpeg" },
    { "db",             "mjpeg" },
    { "pcd",            "photocd" },
    { "pfm",            "pfm" },
    { "pcx",            "pcx" },
    { "png",            "png" },
    { "pns",            "png" },
    { "ptx",            "ptx" },
    { "tga",            "targa" },
    { "tif",            "tiff" },
    { "tiff",           "tiff" },
    { "sgi",            "sgi" },
    { "sun",            "sunrast" },
    { "ras",            "sunrast" },
    { "rs",             "sunrast" },
    { "ra",             "sunrast" },
    { "im1",            "sunrast" },
    { "im8",            "sunrast" },
    { "im24",           "sunrast" },
    { "im32",           "sunrast" },
    { "sunras",         "sunrast" },
    { "xbm",            "xbm" },
    { "pam",            "pam" },
    { "pbm",            "pbm" },
    { "pgm",            "pgm" },
    { "pgmyuv",         "pgmyuv" },
    { "ppm",            "ppm" },
    { "pnm",            "ppm" },
    { "gif",            "gif" }, // usually handled by demux_lavf
    { "pix",            "brender_pix" },
    { "exr",            "exr" },
    { "pic",            "pictor" },
    { "xface",          "xface" },
    { "xwd",            "xwd" },
    {0}
};

const demuxer_desc_t demuxer_desc_mf = {
    .name = "mf",
    .desc = "image files (mf)",
    .read_packet = demux_mf_read_packet,
    .open = demux_open_mf,
    .close = demux_close_mf,
    .seek = demux_seek_mf,
};


CallResult<bool> JSObject::putComputedWithReceiver_RJS(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    Handle<> nameValHandle,
    Handle<> valueHandle,
    Handle<> receiver,
    PropOpFlags opFlags) {
  assert(
      !opFlags.getMustExist() &&
      "mustExist flag cannot be used with computed properties");

  // Try the fast-path first: has "index-like" properties, the "name"
  // already is a valid integer index, selfHandle and receiver are the
  // same, and it is present in storage.
  if (selfHandle->flags_.fastIndexProperties) {
    if (auto arrayIndex = toArrayIndexFastPath(*nameValHandle)) {
      if (selfHandle.getHermesValue().getRaw() == receiver->getRaw()) {
        if (haveOwnIndexed(selfHandle.get(), runtime, *arrayIndex)) {
          auto result =
              setOwnIndexed(selfHandle, runtime, *arrayIndex, valueHandle);
          if (LLVM_UNLIKELY(result == ExecutionStatus::EXCEPTION))
            return ExecutionStatus::EXCEPTION;
          if (LLVM_LIKELY(*result))
            return true;
          if (opFlags.getThrowOnError()) {
            // TODO: better message.
            return runtime->raiseTypeError(
                "Cannot assign to read-only property");
          }
          return false;
        }
      }
    }
  }

  // If nameValHandle is an object, we should convert it to string now,
  // because toString may have side-effect, and we want to do this only
  // once.
  auto converted = toPropertyKeyIfObject(runtime, nameValHandle);
  if (LLVM_UNLIKELY(converted == ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }
  auto nameValPrimitiveHandle = *converted;

  ComputedPropertyDescriptor desc;

  // Look for the property in this object or along the prototype chain.
  MutableHandle<JSObject> propObj{runtime};
  if (LLVM_UNLIKELY(
          getComputedPrimitiveDescriptor(
              selfHandle, runtime, nameValPrimitiveHandle, propObj, desc) ==
          ExecutionStatus::EXCEPTION)) {
    return ExecutionStatus::EXCEPTION;
  }

  // If the property exists (or, we hit a proxy/hostobject on the way
  // up the chain)
  if (propObj) {
    // Get the simple case out of the way: If the property already
    // exists on selfHandle, is not an accessor, selfHandle and
    // receiver are the same, selfHandle is not a host
    // object/proxy/internal setter, and the property is writable,
    // just write into the same slot.

    if (LLVM_LIKELY(
            selfHandle == propObj &&
            selfHandle.getHermesValue().getRaw() == receiver->getRaw() &&
            !desc.flags.accessor && !desc.flags.internalSetter &&
            !desc.flags.hostObject && !desc.flags.proxyObject &&
            desc.flags.writable)) {
      if (LLVM_UNLIKELY(
              setComputedSlotValue(selfHandle, runtime, desc, valueHandle) ==
              ExecutionStatus::EXCEPTION)) {
        return ExecutionStatus::EXCEPTION;
      }
      return true;
    }

    // Is it an accessor?
    if (LLVM_UNLIKELY(desc.flags.accessor)) {
      auto *accessor = vmcast<PropertyAccessor>(
          getComputedSlotValue(propObj.get(), runtime, desc));

      // If it is a read-only accessor, fail.
      if (!accessor->setter) {
        if (opFlags.getThrowOnError()) {
          return runtime->raiseTypeErrorForValue(
              "Cannot assign to property ",
              nameValPrimitiveHandle,
              " which has only a getter");
        }
        return false;
      }

      // Execute the accessor on this object.
      if (accessor->setter.get(runtime)->executeCall1(
              runtime->makeHandle(accessor->setter),
              runtime,
              receiver,
              valueHandle.get()) == ExecutionStatus::EXCEPTION) {
        return ExecutionStatus::EXCEPTION;
      }
      return true;
    }

    if (LLVM_UNLIKELY(desc.flags.proxyObject)) {
      assert(
          !opFlags.getMustExist() &&
          "MustExist cannot be used with Proxy objects");
      CallResult<Handle<>> key = toPropertyKey(runtime, nameValPrimitiveHandle);
      if (key == ExecutionStatus::EXCEPTION)
        return ExecutionStatus::EXCEPTION;
      CallResult<bool> setRes =
          JSProxy::setComputed(propObj, runtime, *key, valueHandle, receiver);
      if (LLVM_UNLIKELY(setRes == ExecutionStatus::EXCEPTION)) {
        return ExecutionStatus::EXCEPTION;
      }
      if (!*setRes && opFlags.getThrowOnError()) {
        // TODO: better message.
        return runtime->raiseTypeError(
            TwineChar16("Proxy trap returned false for property"));
      }
      return setRes;
    }

    if (LLVM_UNLIKELY(!desc.flags.writable)) {
      if (desc.flags.staticBuiltin) {
        SymbolID id{};
        LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);
        return raiseErrorForOverridingStaticBuiltin(
            selfHandle, runtime, runtime->makeHandle(id));
      }
      if (opFlags.getThrowOnError()) {
        return runtime->raiseTypeErrorForValue(
            "Cannot assign to read-only property ", nameValPrimitiveHandle, "");
      }
      return false;
    }

    if (selfHandle == propObj && desc.flags.internalSetter) {
      SymbolID id{};
      LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);
      return internalSetter(
          selfHandle,
          runtime,
          id,
          desc.castToNamedPropertyDescriptorRef(),
          valueHandle,
          opFlags);
    }
  }

  // The property does not exist as an conventional own property on
  // this object.

  MutableHandle<JSObject> receiverHandle{runtime, *selfHandle};
  if (selfHandle.getHermesValue().getRaw() != receiver->getRaw() ||
      receiverHandle->isHostObject() || receiverHandle->isProxyObject()) {
    if (selfHandle.getHermesValue().getRaw() != receiver->getRaw()) {
      receiverHandle = dyn_vmcast<JSObject>(*receiver);
    }
    if (!receiverHandle) {
      return false;
    }
    CallResult<bool> descDefinedRes = getOwnComputedPrimitiveDescriptor(
        receiverHandle, runtime, nameValPrimitiveHandle, IgnoreProxy::No, desc);
    if (LLVM_UNLIKELY(descDefinedRes == ExecutionStatus::EXCEPTION)) {
      return ExecutionStatus::EXCEPTION;
    }
    DefinePropertyFlags dpf;
    if (*descDefinedRes) {
      if (LLVM_UNLIKELY(desc.flags.accessor || !desc.flags.writable)) {
        return false;
      }

      if (LLVM_LIKELY(
              !desc.flags.internalSetter && !receiverHandle->isHostObject() &&
              !receiverHandle->isProxyObject())) {
        if (LLVM_UNLIKELY(
                setComputedSlotValue(
                    receiverHandle, runtime, desc, valueHandle) ==
                ExecutionStatus::EXCEPTION)) {
          return ExecutionStatus::EXCEPTION;
        }
        return true;
      }
    }

    if (LLVM_UNLIKELY(
            desc.flags.internalSetter || receiverHandle->isHostObject() ||
            receiverHandle->isProxyObject())) {
      // If putComputed is called on a proxy whose target's prototype
      // is an array with a propname of 'length', then internalSetter
      // will be true, and the receiver will be a proxy.  In that case,
      // proxy wins.
      if (receiverHandle->isProxyObject()) {
        if (*descDefinedRes) {
          dpf.setValue = 1;
        } else {
          dpf = DefinePropertyFlags::getDefaultNewPropertyFlags();
        }
        return JSProxy::defineOwnProperty(
            receiverHandle,
            runtime,
            nameValPrimitiveHandle,
            dpf,
            valueHandle,
            opFlags);
      }
      SymbolID id{};
      LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);
      if (desc.flags.internalSetter) {
        return internalSetter(
            receiverHandle,
            runtime,
            id,
            desc.castToNamedPropertyDescriptorRef(),
            valueHandle,
            opFlags);
      }
      assert(
          receiverHandle->isHostObject() && "descriptor flags are impossible");
      return vmcast<HostObject>(receiverHandle.get())->set(id, *valueHandle);
    }
  }

  /// Can we add more properties?
  if (LLVM_UNLIKELY(!receiverHandle->isExtensible())) {
    if (opFlags.getThrowOnError()) {
      return runtime->raiseTypeError(
          "cannot add a new property"); // TODO: better message.
    }
    return false;
  }

  // If we have indexed storage we must check whether the property is an index,
  // and if it is, store it in indexed storage.
  if (receiverHandle->flags_.indexedStorage) {
    OptValue<uint32_t> arrayIndex;
    MutableHandle<StringPrimitive> strPrim{runtime};
    TO_ARRAY_INDEX(runtime, nameValPrimitiveHandle, strPrim, arrayIndex);
    if (arrayIndex) {
      // Check whether we need to update array's ".length" property.
      if (auto *array = dyn_vmcast<JSArray>(receiverHandle.get())) {
        if (LLVM_UNLIKELY(*arrayIndex >= JSArray::getLength(array))) {
          auto cr = putNamed_RJS(
              receiverHandle,
              runtime,
              Predefined::getSymbolID(Predefined::length),
              runtime->makeHandle(
                  HermesValue::encodeNumberValue(*arrayIndex + 1)),
              opFlags);
          if (LLVM_UNLIKELY(cr == ExecutionStatus::EXCEPTION))
            return ExecutionStatus::EXCEPTION;
          if (LLVM_UNLIKELY(!*cr))
            return false;
        }
      }

      auto result =
          setOwnIndexed(receiverHandle, runtime, *arrayIndex, valueHandle);
      if (LLVM_UNLIKELY(result == ExecutionStatus::EXCEPTION))
        return ExecutionStatus::EXCEPTION;
      if (LLVM_LIKELY(*result))
        return true;

      if (opFlags.getThrowOnError()) {
        // TODO: better message.
        return runtime->raiseTypeError("Cannot assign to read-only property");
      }
      return false;
    }
  }

  SymbolID id{};
  LAZY_TO_IDENTIFIER(runtime, nameValPrimitiveHandle, id);

  // Add a new named property.
  return addOwnProperty(
      receiverHandle,
      runtime,
      id,
      DefinePropertyFlags::getDefaultNewPropertyFlags(),
      valueHandle,
      opFlags);
}

Handle set where internalSetter and Proxy are both true

Summary:
If putComputed is called on a proxy whose target's prototype
is an array with a propname of 'length', then internalSetter will be
true, and the receiver will be a proxy.  In that case, proxy needs to
win; the behavior may assert or be UB otherwise.

Reviewed By: tmikov

Differential Revision: D23916279

fbshipit-source-id: c760356d48a02ece565fb4bc1acdafd7ccad7c68

/* No explicit global definitions found. */


CallResult<bool> JSObject::putNamedWithReceiver_RJS(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    SymbolID name,
    Handle<> valueHandle,
    Handle<> receiver,
    PropOpFlags opFlags) {
  NamedPropertyDescriptor desc;

  // Look for the property in this object or along the prototype chain.
  JSObject *propObj = getNamedDescriptor(
      selfHandle,
      runtime,
      name,
      PropertyFlags::defaultNewNamedPropertyFlags(),
      desc);

  // If the property exists (or, we hit a proxy/hostobject on the way
  // up the chain)
  if (propObj) {
    // Get the simple case out of the way: If the property already
    // exists on selfHandle, is not an accessor, selfHandle and
    // receiver are the same, selfHandle is not a host
    // object/proxy/internal setter, and the property is writable,
    // just write into the same slot.

    if (LLVM_LIKELY(
            *selfHandle == propObj &&
            selfHandle.getHermesValue().getRaw() == receiver->getRaw() &&
            !desc.flags.accessor && !desc.flags.internalSetter &&
            !desc.flags.hostObject && !desc.flags.proxyObject &&
            desc.flags.writable)) {
      setNamedSlotValue(
          *selfHandle, runtime, desc, valueHandle.getHermesValue());
      return true;
    }

    if (LLVM_UNLIKELY(desc.flags.accessor)) {
      auto *accessor =
          vmcast<PropertyAccessor>(getNamedSlotValue(propObj, runtime, desc));

      // If it is a read-only accessor, fail.
      if (!accessor->setter) {
        if (opFlags.getThrowOnError()) {
          return runtime->raiseTypeError(
              TwineChar16("Cannot assign to property '") +
              runtime->getIdentifierTable().getStringViewForDev(runtime, name) +
              "' which has only a getter");
        }
        return false;
      }

      // Execute the accessor on this object.
      if (accessor->setter.get(runtime)->executeCall1(
              runtime->makeHandle(accessor->setter),
              runtime,
              receiver,
              *valueHandle) == ExecutionStatus::EXCEPTION) {
        return ExecutionStatus::EXCEPTION;
      }
      return true;
    }

    if (LLVM_UNLIKELY(desc.flags.proxyObject)) {
      assert(
          !opFlags.getMustExist() &&
          "MustExist cannot be used with Proxy objects");
      CallResult<bool> setRes = JSProxy::setNamed(
          runtime->makeHandle(propObj), runtime, name, valueHandle, receiver);
      if (LLVM_UNLIKELY(setRes == ExecutionStatus::EXCEPTION)) {
        return ExecutionStatus::EXCEPTION;
      }
      if (!*setRes && opFlags.getThrowOnError()) {
        return runtime->raiseTypeError(
            TwineChar16("Proxy set returned false for property '") +
            runtime->getIdentifierTable().getStringView(runtime, name) + "'");
      }
      return setRes;
    }

    if (LLVM_UNLIKELY(!desc.flags.writable)) {
      if (desc.flags.staticBuiltin) {
        return raiseErrorForOverridingStaticBuiltin(
            selfHandle, runtime, runtime->makeHandle(name));
      }
      if (opFlags.getThrowOnError()) {
        return runtime->raiseTypeError(
            TwineChar16("Cannot assign to read-only property '") +
            runtime->getIdentifierTable().getStringViewForDev(runtime, name) +
            "'");
      }
      return false;
    }

    if (*selfHandle == propObj && desc.flags.internalSetter) {
      return internalSetter(
          selfHandle, runtime, name, desc, valueHandle, opFlags);
    }
  }

  // The property does not exist as an conventional own property on
  // this object.

  MutableHandle<JSObject> receiverHandle{runtime, *selfHandle};
  if (selfHandle.getHermesValue().getRaw() != receiver->getRaw() ||
      receiverHandle->isHostObject() || receiverHandle->isProxyObject()) {
    if (selfHandle.getHermesValue().getRaw() != receiver->getRaw()) {
      receiverHandle = dyn_vmcast<JSObject>(*receiver);
    }
    if (!receiverHandle) {
      return false;
    }

    if (getOwnNamedDescriptor(receiverHandle, runtime, name, desc)) {
      if (LLVM_UNLIKELY(desc.flags.accessor || !desc.flags.writable)) {
        return false;
      }

      assert(
          !receiverHandle->isHostObject() && !receiverHandle->isProxyObject() &&
          "getOwnNamedDescriptor never sets hostObject or proxyObject flags");

      setNamedSlotValue(
          *receiverHandle, runtime, desc, valueHandle.getHermesValue());
      return true;
    }

    // Now deal with host and proxy object cases.  We need to call
    // getOwnComputedPrimitiveDescriptor because it knows how to call
    // the [[getOwnProperty]] Proxy impl if needed.
    if (LLVM_UNLIKELY(
            receiverHandle->isHostObject() ||
            receiverHandle->isProxyObject())) {
      if (receiverHandle->isHostObject()) {
        return vmcast<HostObject>(receiverHandle.get())
            ->set(name, *valueHandle);
      }
      ComputedPropertyDescriptor desc;
      Handle<> nameValHandle = runtime->makeHandle(name);
      CallResult<bool> descDefinedRes = getOwnComputedPrimitiveDescriptor(
          receiverHandle, runtime, nameValHandle, IgnoreProxy::No, desc);
      if (LLVM_UNLIKELY(descDefinedRes == ExecutionStatus::EXCEPTION)) {
        return ExecutionStatus::EXCEPTION;
      }
      DefinePropertyFlags dpf;
      if (*descDefinedRes) {
        dpf.setValue = 1;
      } else {
        dpf = DefinePropertyFlags::getDefaultNewPropertyFlags();
      }
      return JSProxy::defineOwnProperty(
          receiverHandle, runtime, nameValHandle, dpf, valueHandle, opFlags);
    }
  }

  // Does the caller require it to exist?
  if (LLVM_UNLIKELY(opFlags.getMustExist())) {
    return runtime->raiseReferenceError(
        TwineChar16("Property '") +
        runtime->getIdentifierTable().getStringViewForDev(runtime, name) +
        "' doesn't exist");
  }

  // Add a new property.

  return addOwnProperty(
      receiverHandle,
      runtime,
      name,
      DefinePropertyFlags::getDefaultNewPropertyFlags(),
      valueHandle,
      opFlags);
}


CallResult<bool> JSObject::defineOwnComputedPrimitive(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    Handle<> nameValHandle,
    DefinePropertyFlags dpFlags,
    Handle<> valueOrAccessor,
    PropOpFlags opFlags) {
  assert(
      !nameValHandle->isObject() &&
      "nameValHandle passed to "
      "defineOwnComputedPrimitive() cannot be "
      "an object");
  assert(
      !opFlags.getMustExist() && "cannot use mustExist with defineOwnProperty");
  assert(
      !(dpFlags.setValue && dpFlags.isAccessor()) &&
      "Cannot set both value and accessor");
  assert(
      (dpFlags.setValue || dpFlags.isAccessor() ||
       valueOrAccessor.get().isUndefined()) &&
      "value must be undefined when all of setValue/setSetter/setGetter are "
      "false");
  assert(
      !dpFlags.enableInternalSetter &&
      "Cannot set internalSetter on a computed property");
#ifndef NDEBUG
  if (dpFlags.isAccessor()) {
    assert(valueOrAccessor.get().isPointer() && "accessor must be non-empty");
    assert(
        !dpFlags.setWritable && !dpFlags.writable &&
        "writable must not be set with accessors");
  }
#endif

  // If the name is a valid integer array index, store it here.
  OptValue<uint32_t> arrayIndex;

  // If we have indexed storage, we must attempt to convert the name to array
  // index, even if the conversion is expensive.
  if (selfHandle->flags_.indexedStorage) {
    MutableHandle<StringPrimitive> strPrim{runtime};
    TO_ARRAY_INDEX(runtime, nameValHandle, strPrim, arrayIndex);
  }

  SymbolID id{};

  // If not storing a property with an array index name, or if we don't have
  // indexed storage, just pass to the named routine.
  if (!arrayIndex) {
    LAZY_TO_IDENTIFIER(runtime, nameValHandle, id);
    return defineOwnPropertyInternal(
        selfHandle, runtime, id, dpFlags, valueOrAccessor, opFlags);
  }

  // At this point we know that we have indexed storage and that the property
  // has an index-like name.

  // First check if a named property with the same name exists.
  if (selfHandle->clazz_.get(runtime)->getHasIndexLikeProperties()) {
    LAZY_TO_IDENTIFIER(runtime, nameValHandle, id);

    NamedPropertyDescriptor desc;
    auto pos = findProperty(selfHandle, runtime, id, desc);
    // If we found a named property, update it.
    if (pos) {
      return updateOwnProperty(
          selfHandle,
          runtime,
          id,
          *pos,
          desc,
          dpFlags,
          valueOrAccessor,
          opFlags);
    }
  }

  // Does an indexed property with that index exist?
  auto indexedPropPresent =
      getOwnIndexedPropertyFlags(selfHandle.get(), runtime, *arrayIndex);
  if (indexedPropPresent) {
    // The current value of the property.
    HermesValue curValueOrAccessor =
        getOwnIndexed(selfHandle.get(), runtime, *arrayIndex);

    auto updateStatus = checkPropertyUpdate(
        runtime,
        *indexedPropPresent,
        dpFlags,
        curValueOrAccessor,
        valueOrAccessor,
        opFlags);
    if (updateStatus == ExecutionStatus::EXCEPTION)
      return ExecutionStatus::EXCEPTION;
    if (updateStatus->first == PropertyUpdateStatus::failed)
      return false;

    // The property update is valid, but can the property remain an "indexed"
    // property, or do we need to convert it to a named property?
    // If the property flags didn't change, the property remains indexed.
    if (updateStatus->second == *indexedPropPresent) {
      // If the value doesn't change, we are done.
      if (updateStatus->first == PropertyUpdateStatus::done)
        return true;

      // If we successfully updated the value, we are done.
      auto result =
          setOwnIndexed(selfHandle, runtime, *arrayIndex, valueOrAccessor);
      if (LLVM_UNLIKELY(result == ExecutionStatus::EXCEPTION))
        return ExecutionStatus::EXCEPTION;
      if (*result)
        return true;

      if (opFlags.getThrowOnError()) {
        // TODO: better error message.
        return runtime->raiseTypeError(
            "cannot change read-only property value");
      }

      return false;
    }

    // OK, we need to convert an indexed property to a named one.

    // Check whether to use the supplied value, or to reuse the old one, as we
    // are simply reconfiguring it.
    MutableHandle<> value{runtime};
    if (dpFlags.setValue || dpFlags.isAccessor()) {
      value = valueOrAccessor.get();
    } else {
      value = curValueOrAccessor;
    }

    // Update dpFlags to match the existing property flags.
    dpFlags.setEnumerable = 1;
    dpFlags.setWritable = 1;
    dpFlags.setConfigurable = 1;
    dpFlags.enumerable = updateStatus->second.enumerable;
    dpFlags.writable = updateStatus->second.writable;
    dpFlags.configurable = updateStatus->second.configurable;

    // Delete the existing indexed property.
    if (!deleteOwnIndexed(selfHandle, runtime, *arrayIndex)) {
      if (opFlags.getThrowOnError()) {
        // TODO: better error message.
        return runtime->raiseTypeError("Cannot define property");
      }
      return false;
    }

    // Add the new named property.
    LAZY_TO_IDENTIFIER(runtime, nameValHandle, id);
    return addOwnProperty(selfHandle, runtime, id, dpFlags, value, opFlags);
  }

  /// Can we add new properties?
  if (!selfHandle->isExtensible()) {
    if (opFlags.getThrowOnError()) {
      return runtime->raiseTypeError(
          "cannot add a new property"); // TODO: better message.
    }
    return false;
  }

  // This is a new property with an index-like name.
  // Check whether we need to update array's ".length" property.
  bool updateLength = false;
  if (auto arrayHandle = Handle<JSArray>::dyn_vmcast(selfHandle)) {
    if (LLVM_UNLIKELY(*arrayIndex >= JSArray::getLength(*arrayHandle))) {
      NamedPropertyDescriptor lengthDesc;
      bool lengthPresent = getOwnNamedDescriptor(
          arrayHandle,
          runtime,
          Predefined::getSymbolID(Predefined::length),
          lengthDesc);
      (void)lengthPresent;
      assert(lengthPresent && ".length must be present in JSArray");

      if (!lengthDesc.flags.writable) {
        if (opFlags.getThrowOnError()) {
          return runtime->raiseTypeError(
              "Cannot assign to read-only 'length' property of array");
        }
        return false;
      }

      updateLength = true;
    }
  }

  bool newIsIndexed = canNewPropertyBeIndexed(dpFlags);
  if (newIsIndexed) {
    auto result = setOwnIndexed(
        selfHandle,
        runtime,
        *arrayIndex,
        dpFlags.setValue ? valueOrAccessor : Runtime::getUndefinedValue());
    if (LLVM_UNLIKELY(result == ExecutionStatus::EXCEPTION))
      return ExecutionStatus::EXCEPTION;
    if (!*result) {
      if (opFlags.getThrowOnError()) {
        // TODO: better error message.
        return runtime->raiseTypeError("Cannot define property");
      }
      return false;
    }
  }

  // If this is an array and we need to update ".length", do so.
  if (updateLength) {
    // This should always succeed since we are simply enlarging the length.
    auto res = JSArray::setLength(
        Handle<JSArray>::vmcast(selfHandle), runtime, *arrayIndex + 1, opFlags);
    (void)res;
    assert(
        res != ExecutionStatus::EXCEPTION && *res &&
        "JSArray::setLength() failed unexpectedly");
  }

  if (newIsIndexed)
    return true;

  // We are adding a new property with an index-like name.
  LAZY_TO_IDENTIFIER(runtime, nameValHandle, id);
  return addOwnProperty(
      selfHandle, runtime, id, dpFlags, valueOrAccessor, opFlags);
}


CallResult<bool> JSObject::defineOwnPropertyInternal(
    Handle<JSObject> selfHandle,
    Runtime *runtime,
    SymbolID name,
    DefinePropertyFlags dpFlags,
    Handle<> valueOrAccessor,
    PropOpFlags opFlags) {
  assert(
      !opFlags.getMustExist() && "cannot use mustExist with defineOwnProperty");
  assert(
      !(dpFlags.setValue && dpFlags.isAccessor()) &&
      "Cannot set both value and accessor");
  assert(
      (dpFlags.setValue || dpFlags.isAccessor() ||
       valueOrAccessor.get().isUndefined()) &&
      "value must be undefined when all of setValue/setSetter/setGetter are "
      "false");
#ifndef NDEBUG
  if (dpFlags.isAccessor()) {
    assert(valueOrAccessor.get().isPointer() && "accessor must be non-empty");
    assert(
        !dpFlags.setWritable && !dpFlags.writable &&
        "writable must not be set with accessors");
  }
#endif

  // Is it an existing property.
  NamedPropertyDescriptor desc;
  auto pos = findProperty(selfHandle, runtime, name, desc);
  if (pos) {
    return updateOwnProperty(
        selfHandle,
        runtime,
        name,
        *pos,
        desc,
        dpFlags,
        valueOrAccessor,
        opFlags);
  }

  if (LLVM_UNLIKELY(
          selfHandle->flags_.lazyObject || selfHandle->flags_.proxyObject)) {
    if (selfHandle->flags_.proxyObject) {
      return JSProxy::defineOwnProperty(
          selfHandle,
          runtime,
          name.isUniqued() ? runtime->makeHandle(HermesValue::encodeStringValue(
                                 runtime->getStringPrimFromSymbolID(name)))
                           : runtime->makeHandle(name),
          dpFlags,
          valueOrAccessor,
          opFlags);
    }
    assert(selfHandle->flags_.lazyObject && "descriptor flags are impossible");
    // if the property was not found and the object is lazy we need to
    // initialize it and try again.
    JSObject::initializeLazyObject(runtime, selfHandle);
    return defineOwnPropertyInternal(
        selfHandle, runtime, name, dpFlags, valueOrAccessor, opFlags);
  }

  return addOwnProperty(
      selfHandle, runtime, name, dpFlags, valueOrAccessor, opFlags);
}


  explicit LoadAndRemapMatrixOp(OpKernelConstruction* context)
      : OpKernel(context) {
    OP_REQUIRES_OK(context, context->GetAttr("num_rows", &num_rows_));
    OP_REQUIRES_OK(context, context->GetAttr("num_cols", &num_cols_));
    OP_REQUIRES_OK(
        context, context->GetAttr("max_rows_in_memory", &max_rows_in_memory_));
  }

Fix a check fail

PiperOrigin-RevId: 372011072
Change-Id: I1062cfaed0aa16884e9a16312483794d188db76f


Status RemapVectorToMap(const TTypes<const int64>::Vec& remapping,
                        std::vector<bool>* id_present,
                        std::unordered_map<int64, int64>* old_id_to_new_id) {
  id_present->clear();
  id_present->resize(remapping.size(), false);
  for (int i = 0; i < remapping.size(); ++i) {
    const int64 old_id = remapping(i);
    if (old_id < 0) continue;
    (*id_present)[i] = true;
    if (!gtl::InsertIfNotPresent(old_id_to_new_id, old_id, i)) {
      return errors::Unimplemented(
          strings::StrCat("Old ID ", old_id, " is mapped to both new ID ",
                          old_id_to_new_id->at(old_id), " and ", i,
                          ", which is not supported."));
    }
  }
  return Status::OK();
}


  void Compute(OpKernelContext* context) override {
    // Checks what we're remapping and inverts the relevant remapping Tensors to
    // be maps with key = old ID, value = new ID.
    std::unordered_map<int64, int64> old_row_to_new_row_map;
    std::vector<bool> row_id_present;
    const Tensor* row_remapping_t;
    OP_REQUIRES_OK(context, context->input("row_remapping", &row_remapping_t));
    const auto row_remapping = row_remapping_t->vec<int64>();
    OP_REQUIRES(context, row_remapping.size() == num_rows_,
                errors::InvalidArgument(strings::StrCat(
                    "Size of row_remapping is ", row_remapping.size(),
                    " instead of being equal to num_rows=", num_rows_)));
    OP_REQUIRES_OK(context, RemapVectorToMap(row_remapping, &row_id_present,
                                             &old_row_to_new_row_map));

    // Calculates the min/max old row ID that we need to read, to save us from
    // reading some unnecessary slices of the old tensor.
    int64 min_old_row = -1;
    int64 max_old_row = -1;
    for (int i = 0; i < row_remapping.size(); ++i) {
      if (min_old_row < 0 ||
          (row_remapping(i) >= 0 && row_remapping(i) < min_old_row)) {
        min_old_row = row_remapping(i);
      }
      if (max_old_row < 0 ||
          (row_remapping(i) >= 0 && row_remapping(i) > max_old_row)) {
        max_old_row = row_remapping(i);
      }
    }

    // Processes the remapping for columns.
    std::unordered_map<int64, int64> old_col_to_new_col_map;
    std::vector<bool> col_id_present;
    const Tensor* col_remapping_t;
    OP_REQUIRES_OK(context, context->input("col_remapping", &col_remapping_t));
    const auto col_remapping = col_remapping_t->vec<int64>();
    // Note that we always "remap rows", even when the row vocabulary does
    // not change, because partitioning requires a mapping from partitioned
    // Variables to the full checkpoints we load.
    const bool remap_cols = col_remapping.size() > 0;
    if (remap_cols) {
      OP_REQUIRES(
          context, col_remapping.size() == num_cols_,
          errors::InvalidArgument(strings::StrCat(
              "Provided col_remapping, but its size is ", col_remapping.size(),
              " instead of being equal to num_cols=", num_cols_)));
      OP_REQUIRES_OK(context, RemapVectorToMap(col_remapping, &col_id_present,
                                               &old_col_to_new_col_map));
    } else {
      col_id_present.clear();
      col_id_present.resize(num_cols_, true);
    }

    // Processes the checkpoint source and the provided Tensor name.
    const Tensor* ckpt_path_t;
    OP_REQUIRES_OK(context, context->input("ckpt_path", &ckpt_path_t));
    OP_REQUIRES(
        context, ckpt_path_t->NumElements() == 1,
        errors::InvalidArgument("The `ckpt_path` tensor must have exactly one "
                                "element, got tensor of shape ",
                                ckpt_path_t->shape().DebugString()));
    const string& ckpt_path = ckpt_path_t->scalar<tstring>()();
    const Tensor* old_tensor_name_t;
    OP_REQUIRES_OK(context,
                   context->input("old_tensor_name", &old_tensor_name_t));
    const string& old_tensor_name = old_tensor_name_t->scalar<tstring>()();

    LOG(INFO) << "Processing checkpoint : " << ckpt_path;
    BundleReader reader(context->env(), ckpt_path);
    OP_REQUIRES_OK(context, reader.status());

    DataType tensor_type;
    TensorShape tensor_shape;
    OP_REQUIRES_OK(context, reader.LookupDtypeAndShape(
                                old_tensor_name, &tensor_type, &tensor_shape));
    OP_REQUIRES(context, tensor_type == DT_FLOAT,
                errors::InvalidArgument(strings::StrCat(
                    "Tensor ", old_tensor_name, " has invalid type ",
                    DataTypeString(tensor_type), " instead of expected type ",
                    DataTypeString(DT_FLOAT))));
    // This op is limited to loading Tensors of rank 2 (matrices).
    OP_REQUIRES(
        context, tensor_shape.dims() == 2,
        errors::InvalidArgument(strings::StrCat(
            "Tensor ", old_tensor_name, " has shape ",
            tensor_shape.DebugString(), " of invalid rank ",
            tensor_shape.dims(), " instead of expected shape of rank 2.")));

    if (!remap_cols) {
      // TODO(weiho): Consider relaxing this restriction to allow partial column
      // loading (even when no column remapping is specified) if there turns out
      // to be a use case for it.
      OP_REQUIRES(context, num_cols_ == tensor_shape.dim_size(1),
                  errors::InvalidArgument(strings::StrCat(
                      "Tensor ", old_tensor_name, " has shape ",
                      tensor_shape.DebugString(),
                      ", where the size of its 2nd dimension is ",
                      tensor_shape.dim_size(1),
                      " instead of being equal to num_cols=", num_cols_)));
    }

    // Uses TensorSlice to potentially load the old tensor in chunks in case
    // memory usage is a concern.
    std::vector<TensorSlice> tensor_slices;
    TensorSlice slice(tensor_shape.dims());
    if (min_old_row >= 0 && max_old_row >= 0) {
      int64 row_start = min_old_row;
      // TODO(weiho): Given the list of old row IDs of interest (the keys of
      // old_row_to_new_row_map), we could also try something smarter to
      // find some minimal set of covering ranges for the list of old row IDs
      // such that the size of each range is less than max_rows_in_memory_.
      while (row_start <= max_old_row) {
        const int64 slice_length =
            max_rows_in_memory_ <= 0
                // If max_rows_in_memory_ <= 0, we just load the entire chunk.
                ? max_old_row - row_start + 1
                : std::min(max_rows_in_memory_, max_old_row - row_start + 1);
        slice.set_start(0, row_start);
        slice.set_length(0, slice_length);
        tensor_slices.push_back(slice);
        row_start += slice_length;
      }
    }

    // Allocates the output matrix.
    Tensor* output_matrix_t = nullptr;
    OP_REQUIRES_OK(context,
                   context->allocate_output("output_matrix",
                                            TensorShape({num_rows_, num_cols_}),
                                            &output_matrix_t));
    auto output_matrix = output_matrix_t->matrix<float>();

    // Iterates through tensor slices and copies over values from the old tensor
    // to the output matrix.
    int64 row_index = min_old_row;
    int64 rows_copied = 0;
    Tensor loaded_tensor_t;
    for (const TensorSlice& tensor_slice : tensor_slices) {
      LOG(INFO) << "Loading slice " << tensor_slice.DebugString();
      TensorShape slice_shape;
      OP_REQUIRES_OK(context,
                     tensor_slice.SliceTensorShape(tensor_shape, &slice_shape));
      // Potentially re-allocates the tensor buffer since the last slice may
      // have fewer rows than the other slices.
      if (loaded_tensor_t.shape() != slice_shape) {
        loaded_tensor_t = Tensor(DT_FLOAT, slice_shape);
      }
      OP_REQUIRES_OK(context, reader.LookupSlice(old_tensor_name, tensor_slice,
                                                 &loaded_tensor_t));

      // Iterates through the old loaded tensor slice row-by-row.
      for (int row = 0; row < loaded_tensor_t.dim_size(0); ++row, ++row_index) {
        if (row_index % 500000 == min_old_row) {
          LOG(INFO) << "Processing old row " << row_index;
        }

        // If the old row ID is not found in old_row_to_new_row_map, continue
        // to the next row; otherwise, copy it to the output matrix.
        const int64* new_row_ptr =
            gtl::FindOrNull(old_row_to_new_row_map, row_index);
        if (new_row_ptr == nullptr) {
          continue;
        }
        ++rows_copied;
        const int64 new_row = *new_row_ptr;

        // Copies over the row element-by-element, in case remapping is needed
        // along the column axis.
        const auto& loaded_tensor = loaded_tensor_t.matrix<float>();
        for (int old_col = 0; old_col < loaded_tensor_t.dim_size(1);
             ++old_col) {
          int64 new_col = old_col;
          if (remap_cols) {
            const int64* new_col_ptr =
                gtl::FindOrNull(old_col_to_new_col_map, old_col);
            if (new_col_ptr == nullptr) {
              // Column remapping is specified, but this column is not found in
              // old_col_to_new_col_map, so we leave it uninitialized, to be
              // filled in with initializing_values later.
              continue;
            }
            new_col = *new_col_ptr;
          }

          OP_REQUIRES(context,
                      new_row < num_rows_ && new_col < num_cols_ &&
                          new_row >= 0 && new_col >= 0,
                      errors::Internal(strings::StrCat(
                          "new_row=", new_row, " and new_col=", new_col,
                          " should have been less than num_rows_=", num_rows_,
                          " and num_cols_=", num_cols_,
                          " and non-negative. This should never have happened "
                          "if the code were correct. Please file a bug.")));
          output_matrix(new_row, new_col) = loaded_tensor(row, old_col);
        }
      }
    }
    LOG(INFO) << "Copied " << rows_copied << " rows from old matrix (with "
              << tensor_shape.dim_size(0) << " rows) to new matrix (with "
              << num_rows_ << " rows).";

    // At this point, there are potentially whole rows/columns uninitialized
    // (corresponding to the indices where row_id_present/col_id_present are
    // false). We fill this in cell-by-cell using row_id_present and
    // col_id_present while dequeuing from the initializing_values vector.
    const Tensor* initializing_values_t;
    OP_REQUIRES_OK(
        context, context->input("initializing_values", &initializing_values_t));
    const auto initializing_values = initializing_values_t->flat<float>();
    int64 initializing_values_index = 0;
    for (int i = 0; i < num_rows_; ++i) {
      for (int j = 0; j < num_cols_; ++j) {
        if (row_id_present[i] && col_id_present[j]) continue;
        OP_REQUIRES(
            context, initializing_values_index < initializing_values.size(),
            errors::InvalidArgument(
                "initializing_values contained ", initializing_values.size(),
                " elements, but more missing values remain."));
        output_matrix(i, j) = initializing_values(initializing_values_index);
        ++initializing_values_index;
      }
    }

    // Checks that we used all the given initializing values.
    OP_REQUIRES(
        context, initializing_values_index == initializing_values.size(),
        errors::InvalidArgument(
            "initializing_values contained ", initializing_values.size(),
            " elements, but only ", initializing_values_index,
            " elements were used to fill in missing values."));
  }


static int selectExpander(Walker *pWalker, Select *p){
  Parse *pParse = pWalker->pParse;
  int i, j, k;
  SrcList *pTabList;
  ExprList *pEList;
  struct SrcList_item *pFrom;
  sqlite3 *db = pParse->db;
  Expr *pE, *pRight, *pExpr;
  u16 selFlags = p->selFlags;
  u32 elistFlags = 0;

  p->selFlags |= SF_Expanded;
  if( db->mallocFailed  ){
    return WRC_Abort;
  }
  assert( p->pSrc!=0 );
  if( (selFlags & SF_Expanded)!=0 ){
    return WRC_Prune;
  }
  if( pWalker->eCode ){
    /* Renumber selId because it has been copied from a view */
    p->selId = ++pParse->nSelect;
  }
  pTabList = p->pSrc;
  pEList = p->pEList;
  sqlite3WithPush(pParse, p->pWith, 0);

  /* Make sure cursor numbers have been assigned to all entries in
  ** the FROM clause of the SELECT statement.
  */
  sqlite3SrcListAssignCursors(pParse, pTabList);

  /* Look up every table named in the FROM clause of the select.  If
  ** an entry of the FROM clause is a subquery instead of a table or view,
  ** then create a transient table structure to describe the subquery.
  */
  for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
    Table *pTab;
    assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 );
    if( pFrom->fg.isRecursive ) continue;
    assert( pFrom->pTab==0 );
#ifndef SQLITE_OMIT_CTE
    if( withExpand(pWalker, pFrom) ) return WRC_Abort;
    if( pFrom->pTab ) {} else
#endif
    if( pFrom->zName==0 ){
#ifndef SQLITE_OMIT_SUBQUERY
      Select *pSel = pFrom->pSelect;
      /* A sub-query in the FROM clause of a SELECT */
      assert( pSel!=0 );
      assert( pFrom->pTab==0 );
      if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort;
      if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort;
#endif
    }else{
      /* An ordinary table or view name in the FROM clause */
      assert( pFrom->pTab==0 );
      pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom);
      if( pTab==0 ) return WRC_Abort;
      if( pTab->nTabRef>=0xffff ){
        sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535",
           pTab->zName);
        pFrom->pTab = 0;
        return WRC_Abort;
      }
      pTab->nTabRef++;
      if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){
        return WRC_Abort;
      }
#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)
      if( IsVirtual(pTab) || pTab->pSelect ){
        i16 nCol;
        u8 eCodeOrig = pWalker->eCode;
        if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort;
        assert( pFrom->pSelect==0 );
        if( pTab->pSelect && (db->flags & SQLITE_EnableView)==0 ){
          sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited",
            pTab->zName);
        }
#ifndef SQLITE_OMIT_VIRTUALTABLE
        if( IsVirtual(pTab)
         && pFrom->fg.fromDDL
         && ALWAYS(pTab->pVTable!=0)
         && pTab->pVTable->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0)
        ){
          sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"",
                                  pTab->zName);
        }
#endif
        pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0);
        nCol = pTab->nCol;
        pTab->nCol = -1;
        pWalker->eCode = 1;  /* Turn on Select.selId renumbering */
        sqlite3WalkSelect(pWalker, pFrom->pSelect);
        pWalker->eCode = eCodeOrig;
        pTab->nCol = nCol;
      }
#endif
    }

    /* Locate the index named by the INDEXED BY clause, if any. */
    if( sqlite3IndexedByLookup(pParse, pFrom) ){
      return WRC_Abort;
    }
  }

  /* Process NATURAL keywords, and ON and USING clauses of joins.
  */
  if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){
    return WRC_Abort;
  }

  /* For every "*" that occurs in the column list, insert the names of
  ** all columns in all tables.  And for every TABLE.* insert the names
  ** of all columns in TABLE.  The parser inserted a special expression
  ** with the TK_ASTERISK operator for each "*" that it found in the column
  ** list.  The following code just has to locate the TK_ASTERISK
  ** expressions and expand each one to the list of all columns in
  ** all tables.
  **
  ** The first loop just checks to see if there are any "*" operators
  ** that need expanding.
  */
  for(k=0; k<pEList->nExpr; k++){
    pE = pEList->a[k].pExpr;
    if( pE->op==TK_ASTERISK ) break;
    assert( pE->op!=TK_DOT || pE->pRight!=0 );
    assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) );
    if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break;
    elistFlags |= pE->flags;
  }
  if( k<pEList->nExpr ){
    /*
    ** If we get here it means the result set contains one or more "*"
    ** operators that need to be expanded.  Loop through each expression
    ** in the result set and expand them one by one.
    */
    struct ExprList_item *a = pEList->a;
    ExprList *pNew = 0;
    int flags = pParse->db->flags;
    int longNames = (flags & SQLITE_FullColNames)!=0
                      && (flags & SQLITE_ShortColNames)==0;

    for(k=0; k<pEList->nExpr; k++){
      pE = a[k].pExpr;
      elistFlags |= pE->flags;
      pRight = pE->pRight;
      assert( pE->op!=TK_DOT || pRight!=0 );
      if( pE->op!=TK_ASTERISK
       && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK)
      ){
        /* This particular expression does not need to be expanded.
        */
        pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr);
        if( pNew ){
          pNew->a[pNew->nExpr-1].zEName = a[k].zEName;
          pNew->a[pNew->nExpr-1].eEName = a[k].eEName;
          a[k].zEName = 0;
        }
        a[k].pExpr = 0;
      }else{
        /* This expression is a "*" or a "TABLE.*" and needs to be
        ** expanded. */
        int tableSeen = 0;      /* Set to 1 when TABLE matches */
        char *zTName = 0;       /* text of name of TABLE */
        if( pE->op==TK_DOT ){
          assert( pE->pLeft!=0 );
          assert( !ExprHasProperty(pE->pLeft, EP_IntValue) );
          zTName = pE->pLeft->u.zToken;
        }
        for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
          Table *pTab = pFrom->pTab;
          Select *pSub = pFrom->pSelect;
          char *zTabName = pFrom->zAlias;
          const char *zSchemaName = 0;
          int iDb;
          if( zTabName==0 ){
            zTabName = pTab->zName;
          }
          if( db->mallocFailed ) break;
          if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){
            pSub = 0;
            if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){
              continue;
            }
            iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
            zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*";
          }
          for(j=0; j<pTab->nCol; j++){
            char *zName = pTab->aCol[j].zName;
            char *zColname;  /* The computed column name */
            char *zToFree;   /* Malloced string that needs to be freed */
            Token sColname;  /* Computed column name as a token */

            assert( zName );
            if( zTName && pSub
             && sqlite3MatchEName(&pSub->pEList->a[j], 0, zTName, 0)==0
            ){
              continue;
            }

            /* If a column is marked as 'hidden', omit it from the expanded
            ** result-set list unless the SELECT has the SF_IncludeHidden
            ** bit set.
            */
            if( (p->selFlags & SF_IncludeHidden)==0
             && IsHiddenColumn(&pTab->aCol[j])
            ){
              continue;
            }
            tableSeen = 1;

            if( i>0 && zTName==0 ){
              if( (pFrom->fg.jointype & JT_NATURAL)!=0
                && tableAndColumnIndex(pTabList, i, zName, 0, 0, 1)
              ){
                /* In a NATURAL join, omit the join columns from the
                ** table to the right of the join */
                continue;
              }
              if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){
                /* In a join with a USING clause, omit columns in the
                ** using clause from the table on the right. */
                continue;
              }
            }
            pRight = sqlite3Expr(db, TK_ID, zName);
            zColname = zName;
            zToFree = 0;
            if( longNames || pTabList->nSrc>1 ){
              Expr *pLeft;
              pLeft = sqlite3Expr(db, TK_ID, zTabName);
              pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight);
              if( zSchemaName ){
                pLeft = sqlite3Expr(db, TK_ID, zSchemaName);
                pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr);
              }
              if( longNames ){
                zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName);
                zToFree = zColname;
              }
            }else{
              pExpr = pRight;
            }
            pNew = sqlite3ExprListAppend(pParse, pNew, pExpr);
            sqlite3TokenInit(&sColname, zColname);
            sqlite3ExprListSetName(pParse, pNew, &sColname, 0);
            if( pNew && (p->selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){
              struct ExprList_item *pX = &pNew->a[pNew->nExpr-1];
              sqlite3DbFree(db, pX->zEName);
              if( pSub ){
                pX->zEName = sqlite3DbStrDup(db, pSub->pEList->a[j].zEName);
                testcase( pX->zEName==0 );
              }else{
                pX->zEName = sqlite3MPrintf(db, "%s.%s.%s",
                                           zSchemaName, zTabName, zColname);
                testcase( pX->zEName==0 );
              }
              pX->eEName = ENAME_TAB;
            }
            sqlite3DbFree(db, zToFree);
          }
        }
        if( !tableSeen ){
          if( zTName ){
            sqlite3ErrorMsg(pParse, "no such table: %s", zTName);
          }else{
            sqlite3ErrorMsg(pParse, "no tables specified");
          }
        }
      }
    }
    sqlite3ExprListDelete(db, pEList);
    p->pEList = pNew;
  }
  if( p->pEList ){
    if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){
      sqlite3ErrorMsg(pParse, "too many columns in result set");
      return WRC_Abort;
    }
    if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){
      p->selFlags |= SF_ComplexResult;
    }
  }
  return WRC_Continue;
}

Fix a problem with ALTER TABLE for views that have a nested FROM clause.
Ticket [f50af3e8a565776b].

FossilOrigin-Name: c431b3fd8fd0f6a6974bba3e9366b0430ec003d570e7ce70ceefbcff5fe4b6fa


void CalculateUsedRange(const Tensor& input, qint32* used_min_quantized,
                        qint32* used_max_quantized) {
  auto input_array = input.flat<qint32>();
  Eigen::Tensor<qint32, 0, Eigen::RowMajor> min = input_array.minimum();
  Eigen::Tensor<qint32, 0, Eigen::RowMajor> max = input_array.maximum();
  *used_min_quantized = min();
  *used_max_quantized = max();
}

Fix out of bound read in requantization_range_op.cc

PiperOrigin-RevId: 372129031
Change-Id: Ie684ab98a3840c5186ead3eafffc0e0ed0e8030d


explicit RequantizationRangeOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}


  void Compute(OpKernelContext* ctx) override {
    const Tensor& input = ctx->input(0);
    OP_REQUIRES(ctx, ctx->input(1).NumElements() > 0,
                errors::InvalidArgument("Input min must not be empty."));
    OP_REQUIRES(ctx, ctx->input(2).NumElements() > 0,
                errors::InvalidArgument("Input max must not be empty."));
    const float input_min_float = ctx->input(1).flat<float>()(0);
    const float input_max_float = ctx->input(2).flat<float>()(0);
    Tensor* output_min = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &output_min));
    Tensor* output_max = nullptr;
    OP_REQUIRES_OK(ctx, ctx->allocate_output(1, TensorShape({}), &output_max));

    qint32 used_min_quantized;
    qint32 used_max_quantized;
    CalculateUsedRange(input, &used_min_quantized, &used_max_quantized);

    // We want to make sure that the minimum is no larger than zero, so that the
    // convolution operation can run efficiently.
    const float used_min_float = std::min(
        0.0f,
        QuantizedToFloat(used_min_quantized, input_min_float, input_max_float));
    const float used_max_float =
        QuantizedToFloat(used_max_quantized, input_min_float, input_max_float);

    output_min->flat<float>().setConstant(used_min_float);
    output_max->flat<float>().setConstant(used_max_float);
  }


static int Get16m(const void * Short)
{
    return (((uchar *)Short)[0] << 8) | ((uchar *)Short)[1];
}

Same error as previous checking in different function

static Section_t * Sections = NULL;

static int SectionsAllocated;

static int SectionsRead;

static int HaveAll;


void DiscardData(void)
{
    int a;

    for (a=0;a<SectionsRead;a++){
        free(Sections[a].Data);
    }

    memset(&ImageInfo, 0, sizeof(ImageInfo));
    SectionsRead = 0;
    HaveAll = 0;
}

ImageInfo_t ImageInfo;

static Section_t * Sections = NULL;

static int SectionsAllocated;

static int SectionsRead;

static int HaveAll;


int ReadJpegFile(const char * FileName, ReadMode_t ReadMode)
{
    FILE * infile;
    int ret;

    infile = fopen(FileName, "rb"); // Unix ignores 'b', windows needs it.

    if (infile == NULL) {
        fprintf(stderr, "can't open '%s'\n", FileName);
        return FALSE;
    }

    // Scan the JPEG headers.
    ret = ReadJpegSections(infile, ReadMode);
    if (!ret){
        if (ReadMode == READ_ANY){
            // Process any files mode.  Ignore the fact that it's not
            // a jpeg file.
            ret = TRUE;
        }else{
            fprintf(stderr,"Not JPEG: %s\n",FileName);
        }
    }

    fclose(infile);

    if (ret == FALSE){
        DiscardData();
    }
    return ret;
}


int RemoveSectionType(int SectionType)
{
    int a;
    int retval = FALSE;

    for (a=0;a<SectionsRead-1;a++){
        if (Sections[a].Type == SectionType){
            // Free up this section
            free (Sections[a].Data);
            // Move succeding sections back by one to close space in array.
            memmove(Sections+a, Sections+a+1, sizeof(Section_t) * (SectionsRead-a-1));
            SectionsRead -= 1;
            a -= 1;
            retval = TRUE;
        }
    }
    return retval;
}


void DiscardAllButExif(void)
{
    Section_t ExifKeeper;
    Section_t CommentKeeper;
    Section_t IptcKeeper;
    Section_t XmpKeeper;
    int a;

    memset(&ExifKeeper, 0, sizeof(ExifKeeper));
    memset(&CommentKeeper, 0, sizeof(CommentKeeper));
    memset(&IptcKeeper, 0, sizeof(IptcKeeper));
    memset(&XmpKeeper, 0, sizeof(IptcKeeper));

    for (a=0;a<SectionsRead;a++){
        if (Sections[a].Type == M_EXIF && ExifKeeper.Type == 0){
           ExifKeeper = Sections[a];
        }else if (Sections[a].Type == M_XMP && XmpKeeper.Type == 0){
           XmpKeeper = Sections[a];
        }else if (Sections[a].Type == M_COM && CommentKeeper.Type == 0){
            CommentKeeper = Sections[a];
        }else if (Sections[a].Type == M_IPTC && IptcKeeper.Type == 0){
            IptcKeeper = Sections[a];
        }else{
            free(Sections[a].Data);
        }
    }
    SectionsRead = 0;
    if (ExifKeeper.Type){
        CheckSectionsAllocated();
        Sections[SectionsRead++] = ExifKeeper;
    }
    if (CommentKeeper.Type){
        CheckSectionsAllocated();
        Sections[SectionsRead++] = CommentKeeper;
    }
    if (IptcKeeper.Type){
        CheckSectionsAllocated();
        Sections[SectionsRead++] = IptcKeeper;
    }

    if (XmpKeeper.Type){
        CheckSectionsAllocated();
        Sections[SectionsRead++] = XmpKeeper;
    }
}


Section_t * CreateSection(int SectionType, unsigned char * Data, int Size)
{
    Section_t * NewSection;
    int a;
    int NewIndex;

    NewIndex = 0; // Figure out where to put the comment section.
    if (SectionType == M_EXIF){
        // Exif alwas goes first!
    }else{
        for (;NewIndex < 3;NewIndex++){ // Maximum fourth position (just for the heck of it)
            if (Sections[NewIndex].Type == M_JFIF) continue; // Put it after Jfif
            if (Sections[NewIndex].Type == M_EXIF) continue; // Put it after Exif
            break;
        }
    }

    if (SectionsRead < NewIndex){
        ErrFatal("Too few sections!");
    }

    CheckSectionsAllocated();
    for (a=SectionsRead;a>NewIndex;a--){
        Sections[a] = Sections[a-1];
    }
    SectionsRead += 1;

    NewSection = Sections+NewIndex;

    NewSection->Type = SectionType;
    NewSection->Size = Size;
    NewSection->Data = Data;

    return NewSection;
}


static void process_COM (const uchar * Data, int length)
{
    int ch;
    char Comment[MAX_COMMENT_SIZE+1];
    int nch;
    int a;

    nch = 0;

    if (length > MAX_COMMENT_SIZE) length = MAX_COMMENT_SIZE; // Truncate if it won't fit in our structure.

    for (a=2;a<length;a++){
        ch = Data[a];

        if (ch == '\r' && a < length-1 && Data[a+1] == '\n') continue; // Remove cr followed by lf.

        if (ch >= 32 || ch == '\n' || ch == '\t'){
            Comment[nch++] = (char)ch;
        }else{
            Comment[nch++] = '?';
        }
    }

    Comment[nch] = '\0'; // Null terminate

    if (ShowTags){
        printf("COM marker comment: %s\n",Comment);
    }

    strcpy(ImageInfo.Comments,Comment);
}


int SaveThumbnail(char * ThumbFileName)
{
    FILE * ThumbnailFile;

    if (ImageInfo.ThumbnailOffset == 0 || ImageInfo.ThumbnailSize == 0){
        fprintf(stderr,"Image contains no thumbnail\n");
        return FALSE;
    }

    if (strcmp(ThumbFileName, "-") == 0){
        // A filename of '-' indicates thumbnail goes to stdout.
        // This doesn't make much sense under Windows, so this feature is unix only.
        ThumbnailFile = stdout;
    }else{
        ThumbnailFile = fopen(ThumbFileName,"wb");
    }

    if (ThumbnailFile){
        uchar * ThumbnailPointer;
        Section_t * ExifSection;
        ExifSection = FindSection(M_EXIF);
        ThumbnailPointer = ExifSection->Data+ImageInfo.ThumbnailOffset+8;

        fwrite(ThumbnailPointer, ImageInfo.ThumbnailSize ,1, ThumbnailFile);
        fclose(ThumbnailFile);
        return TRUE;
    }else{
        ErrFatal("Could not write thumbnail file");
        return FALSE;
    }
}


int ReplaceThumbnail(const char * ThumbFileName)
{
    FILE * ThumbnailFile;
    int ThumbLen, NewExifSize;
    Section_t * ExifSection;
    uchar * ThumbnailPointer;

    if (ImageInfo.ThumbnailOffset == 0 || ImageInfo.ThumbnailAtEnd == FALSE){
        if (ThumbFileName == NULL){
            // Delete of nonexistent thumbnail (not even pointers present)
            // No action, no error.
            return FALSE;
        }

        // Adding or removing of thumbnail is not possible - that would require rearranging
        // of the exif header, which is risky, and jhad doesn't know how to do.
        fprintf(stderr,"Image contains no thumbnail to replace - add is not possible\n");
        return FALSE;
    }

    if (ThumbFileName){
        ThumbnailFile = fopen(ThumbFileName,"rb");

        if (ThumbnailFile == NULL){
            noread:
            ErrFatal("Could not read thumbnail file");
            return FALSE;
        }

        // get length
        fseek(ThumbnailFile, 0, SEEK_END);

        ThumbLen = ftell(ThumbnailFile);
        fseek(ThumbnailFile, 0, SEEK_SET);

        if (ThumbLen + ImageInfo.ThumbnailOffset > 0x10000-20){
            ErrFatal("Thumbnail is too large to insert into exif header");
        }
    }else{
        if (ImageInfo.ThumbnailSize == 0){
             return FALSE;
        }

        ThumbLen = 0;

        ThumbnailFile = NULL;
    }

    ExifSection = FindSection(M_EXIF);

    NewExifSize = ImageInfo.ThumbnailOffset+8+ThumbLen;
    ExifSection->Data = (uchar *)realloc(ExifSection->Data, NewExifSize);

    ThumbnailPointer = ExifSection->Data+ImageInfo.ThumbnailOffset+8;

    if (ThumbnailFile){
        if (fread(ThumbnailPointer, 1, ThumbLen, ThumbnailFile) != ThumbLen){
            goto noread;
        }
        fclose(ThumbnailFile);
    }

    ImageInfo.ThumbnailSize = ThumbLen;

    Put32u(ExifSection->Data+ImageInfo.ThumbnailSizeOffset+8, ThumbLen);

    ExifSection->Data[0] = (uchar)(NewExifSize >> 8);
    ExifSection->Data[1] = (uchar)NewExifSize;
    ExifSection->Size = NewExifSize;

    return TRUE;
}


int ReadJpegSections (FILE * infile, ReadMode_t ReadMode)
{
    int a;
    int HaveCom = FALSE;

    a = fgetc(infile);

    if (a != 0xff || fgetc(infile) != M_SOI){
        return FALSE;
    }

    ImageInfo.JfifHeader.XDensity = ImageInfo.JfifHeader.YDensity = 300;
    ImageInfo.JfifHeader.ResolutionUnits = 1;

    for(;;){
        int itemlen;
        int prev;
        int marker = 0;
        int ll,lh, got;
        uchar * Data;

        CheckSectionsAllocated();

        prev = 0;
        for (a=0;;a++){
            marker = fgetc(infile);
            if (marker != 0xff && prev == 0xff) break;
            if (marker == EOF){
                ErrFatal("Unexpected end of file");
            }
            prev = marker;
        }

        if (a > 10){
            ErrNonfatal("Extraneous %d padding bytes before section %02X",a-1,marker);
        }

        Sections[SectionsRead].Type = marker;

        // Read the length of the section.
        lh = fgetc(infile);
        ll = fgetc(infile);
        if (lh == EOF || ll == EOF){
            ErrFatal("Unexpected end of file");
        }

        itemlen = (lh << 8) | ll;

        if (itemlen < 2){
            ErrFatal("invalid marker");
        }

        Sections[SectionsRead].Size = itemlen;

        // Allocate an extra 20 bytes more than needed, because sometimes when reading structures,
        // if the section erroneously ends before short structures that should be there, that can trip
        // memory checkers in combination with fuzzers.
        Data = (uchar *)malloc(itemlen+20);
        if (Data == NULL){
            ErrFatal("Could not allocate memory");
        }
        Sections[SectionsRead].Data = Data;

        // Store first two pre-read bytes.
        Data[0] = (uchar)lh;
        Data[1] = (uchar)ll;

        got = fread(Data+2, 1, itemlen-2, infile); // Read the whole section.
        if (got != itemlen-2){
            ErrFatal("Premature end of file?");
        }
        SectionsRead += 1;

        switch(marker){

            case M_SOS:   // stop before hitting compressed data
                // If reading entire image is requested, read the rest of the data.
                if (ReadMode & READ_IMAGE){
                    int cp, ep, size;
                    // Determine how much file is left.
                    cp = ftell(infile);
                    fseek(infile, 0, SEEK_END);
                    ep = ftell(infile);
                    fseek(infile, cp, SEEK_SET);

                    size = ep-cp;
                    Data = (uchar *)malloc(size);
                    if (Data == NULL){
                        ErrFatal("could not allocate data for entire image");
                    }

                    got = fread(Data, 1, size, infile);
                    if (got != size){
                        ErrFatal("could not read the rest of the image");
                    }

                    CheckSectionsAllocated();
                    Sections[SectionsRead].Data = Data;
                    Sections[SectionsRead].Size = size;
                    Sections[SectionsRead].Type = PSEUDO_IMAGE_MARKER;
                    SectionsRead ++;
                    HaveAll = 1;
                }
                return TRUE;

            case M_DQT:
                // Use for jpeg quality guessing
                process_DQT(Data, itemlen);
                break;

            case M_DHT:
                // Use for jpeg quality guessing
                process_DHT(Data, itemlen);
                break;

            case M_EOI:   // in case it's a tables-only JPEG stream
                fprintf(stderr,"No image in jpeg!\n");
                return FALSE;

            case M_COM: // Comment section
                if (HaveCom || ((ReadMode & READ_METADATA) == 0)){
                    // Discard this section.
                    free(Sections[--SectionsRead].Data);
                }else{
                    process_COM(Data, itemlen);
                    HaveCom = TRUE;
                }
                break;

            case M_JFIF:
                // Regular jpegs always have this tag, exif images have the exif
                // marker instead, althogh ACDsee will write images with both markers.
                // this program will re-create this marker on absence of exif marker.
                // hence no need to keep the copy from the file.
                if (itemlen < 16){
                    fprintf(stderr,"Jfif header too short\n");
                    goto ignore;
                }
                if (memcmp(Data+2, "JFIF\0",5)){
                    fprintf(stderr,"Header missing JFIF marker\n");
                }

                ImageInfo.JfifHeader.Present = TRUE;
                ImageInfo.JfifHeader.ResolutionUnits = Data[9];
                ImageInfo.JfifHeader.XDensity = (Data[10]<<8) | Data[11];
                ImageInfo.JfifHeader.YDensity = (Data[12]<<8) | Data[13];
                if (ShowTags){
                    printf("JFIF SOI marker: Units: %d ",ImageInfo.JfifHeader.ResolutionUnits);
                    switch(ImageInfo.JfifHeader.ResolutionUnits){
                        case 0: printf("(aspect ratio)"); break;
                        case 1: printf("(dots per inch)"); break;
                        case 2: printf("(dots per cm)"); break;
                        default: printf("(unknown)"); break;
                    }
                    printf("  X-density=%d Y-density=%d\n",ImageInfo.JfifHeader.XDensity, ImageInfo.JfifHeader.YDensity);

                    if (Data[14] || Data[15]){
                        fprintf(stderr,"Ignoring jfif header thumbnail\n");
                    }
                }

                ignore:

                free(Sections[--SectionsRead].Data);
                break;

            case M_EXIF:
                // There can be different section using the same marker.
                if (ReadMode & READ_METADATA){
                    if (memcmp(Data+2, "Exif", 4) == 0){
                        process_EXIF(Data, itemlen);
                        break;
                    }else if (memcmp(Data+2, "http:", 5) == 0){
                        Sections[SectionsRead-1].Type = M_XMP; // Change tag for internal purposes.
                        if (ShowTags){
                            printf("Image contains XMP section, %d bytes long\n", itemlen);
                            if (ShowTags){
                                ShowXmp(Sections[SectionsRead-1]);
                            }
                        }
                        break;
                    }
                }
                // Oterwise, discard this section.
                free(Sections[--SectionsRead].Data);
                break;

            case M_IPTC:
                if (ReadMode & READ_METADATA){
                    if (ShowTags){
                        printf("Image contains IPTC section, %d bytes long\n", itemlen);
                    }
                    // Note: We just store the IPTC section.  Its relatively straightforward
                    // and we don't act on any part of it, so just display it at parse time.
                }else{
                    free(Sections[--SectionsRead].Data);
                }
                break;

            case M_SOF0:
            case M_SOF1:
            case M_SOF2:
            case M_SOF3:
            case M_SOF5:
            case M_SOF6:
            case M_SOF7:
            case M_SOF9:
            case M_SOF10:
            case M_SOF11:
            case M_SOF13:
            case M_SOF14:
            case M_SOF15:
                if (itemlen < 8){
                    fprintf(stderr,"Section too short\n");
                    break;
                }
                process_SOFn(Data, marker);
                break;
            default:
                // Skip any other sections.
                if (ShowTags){
                    printf("Jpeg section marker 0x%02x size %d\n",marker, itemlen);
                }
                break;
        }
    }
    return TRUE;
}

ImageInfo_t ImageInfo;

static Section_t * Sections = NULL;

static int SectionsAllocated;

static int SectionsRead;

static int HaveAll;

#define PSEUDO_IMAGE_MARKER 0x123; // Extra value.


void WriteJpegFile(const char * FileName)
{
    FILE * outfile;
    int a;

    if (!HaveAll){
        ErrFatal("Can't write back - didn't read all");
    }

    outfile = fopen(FileName,"wb");
    if (outfile == NULL){
        ErrFatal("Could not open file for write");
    }

    // Initial static jpeg marker.
    fputc(0xff,outfile);
    fputc(0xd8,outfile);

    if (Sections[0].Type != M_EXIF && Sections[0].Type != M_JFIF){
        // The image must start with an exif or jfif marker.  If we threw those away, create one.
        static uchar JfifHead[18] = {
            0xff, M_JFIF,
            0x00, 0x10, 'J' , 'F' , 'I' , 'F' , 0x00, 0x01,
            0x01, 0x01, 0x01, 0x2C, 0x01, 0x2C, 0x00, 0x00
        };

        if (ImageInfo.ResolutionUnit == 2 || ImageInfo.ResolutionUnit == 3){
            // Use the exif resolution info to fill out the jfif header.
            // Usually, for exif images, there's no jfif header, so if wediscard
            // the exif header, use info from the exif header for the jfif header.

            ImageInfo.JfifHeader.ResolutionUnits = (char)(ImageInfo.ResolutionUnit-1);
            // Jfif is 1 and 2, Exif is 2 and 3 for In and cm respecively
            ImageInfo.JfifHeader.XDensity = (int)ImageInfo.xResolution;
            ImageInfo.JfifHeader.YDensity = (int)ImageInfo.yResolution;
        }

        JfifHead[11] = ImageInfo.JfifHeader.ResolutionUnits;
        JfifHead[12] = (uchar)(ImageInfo.JfifHeader.XDensity >> 8);
        JfifHead[13] = (uchar)ImageInfo.JfifHeader.XDensity;
        JfifHead[14] = (uchar)(ImageInfo.JfifHeader.YDensity >> 8);
        JfifHead[15] = (uchar)ImageInfo.JfifHeader.YDensity;

        fwrite(JfifHead, 18, 1, outfile);

        // use the values from the exif data for the jfif header, if we have found values
        if (ImageInfo.ResolutionUnit != 0) {
            // JFIF.ResolutionUnit is {1,2}, EXIF.ResolutionUnit is {2,3}
            JfifHead[11] = (uchar)ImageInfo.ResolutionUnit - 1;
        }
        if (ImageInfo.xResolution > 0.0 && ImageInfo.yResolution > 0.0) {
            JfifHead[12] = (uchar)((int)ImageInfo.xResolution>>8);
            JfifHead[13] = (uchar)((int)ImageInfo.xResolution);

            JfifHead[14] = (uchar)((int)ImageInfo.yResolution>>8);
            JfifHead[15] = (uchar)((int)ImageInfo.yResolution);
        }
    }

    // Write all the misc sections
    for (a=0;a<SectionsRead-1;a++){
        fputc(0xff,outfile);
        fputc((unsigned char)Sections[a].Type, outfile);
        fwrite(Sections[a].Data, Sections[a].Size, 1, outfile);
    }

    // Write the remaining image data.
    fwrite(Sections[a].Data, Sections[a].Size, 1, outfile);

    fclose(outfile);
}


static void process_SOFn (const uchar * Data, int marker)
{
    int data_precision, num_components;

    data_precision = Data[2];
    ImageInfo.Height = Get16m(Data+3);
    ImageInfo.Width = Get16m(Data+5);
    num_components = Data[7];

    if (num_components == 3){
        ImageInfo.IsColor = 1;
    }else{
        ImageInfo.IsColor = 0;
    }

    ImageInfo.Process = marker;

    if (ShowTags){
        printf("JPEG image is %uw * %uh, %d color components, %d bits per sample\n",
                   ImageInfo.Width, ImageInfo.Height, num_components, data_precision);
    }
}


Section_t * FindSection(int SectionType)
{
    int a;

    for (a=0;a<SectionsRead;a++){
        if (Sections[a].Type == SectionType){
            return &Sections[a];
        }
    }
    // Could not be found.
    return NULL;
}


void ResetJpgfile(void)
{
    if (Sections == NULL){
        Sections = (Section_t *)malloc(sizeof(Section_t)*5);
        SectionsAllocated = 5;
    }

    SectionsRead = 0;
    HaveAll = 0;
}


static void CheckSectionsAllocated(void)
{
    if (SectionsRead > SectionsAllocated){
        ErrFatal("allocation screwup");
    }
    if (SectionsRead >= SectionsAllocated){
        SectionsAllocated += SectionsAllocated/2;
        Sections = (Section_t *)realloc(Sections, sizeof(Section_t)*SectionsAllocated);
        if (Sections == NULL){
            ErrFatal("could not allocate data for entire image");
        }
    }
}


int RemoveUnknownSections(void)
{
    int a;
    int Modified = FALSE;
    for (a=0;a<SectionsRead-1;){
        switch(Sections[a].Type){
            case  M_SOF0:
            case  M_SOF1:
            case  M_SOF2:
            case  M_SOF3:
            case  M_SOF5:
            case  M_SOF6:
            case  M_SOF7:
            case  M_SOF9:
            case  M_SOF10:
            case  M_SOF11:
            case  M_SOF13:
            case  M_SOF14:
            case  M_SOF15:
            case  M_SOI:
            case  M_EOI:
            case  M_SOS:
            case  M_JFIF:
            case  M_EXIF:
            case  M_XMP:
            case  M_COM:
            case  M_DQT:
            case  M_DHT:
            case  M_DRI:
            case  M_IPTC:
                // keep.
                a++;
                break;
            default:
                // Unknown.  Delete.
                free (Sections[a].Data);
                // Move succeding sections back by one to close space in array.
                memmove(Sections+a, Sections+a+1, sizeof(Section_t) * (SectionsRead-a-1));
                SectionsRead -= 1;
                Modified = TRUE;
        }
    }
    return Modified;
}


grantpt (int fd)
{
  int retval = -1;
#ifdef PATH_MAX
  char _buf[PATH_MAX];
#else
  char _buf[512];
#endif
  char *buf = _buf;
  struct stat64 st;

  if (__builtin_expect (pts_name (fd, &buf, sizeof (_buf), &st), 0))
    {
      int save_errno = errno;

      /* Check, if the file descriptor is valid.  pts_name returns the
	 wrong errno number, so we cannot use that.  */
      if (__libc_fcntl (fd, F_GETFD) == -1 && errno == EBADF)
	return -1;

       /* If the filedescriptor is no TTY, grantpt has to set errno
	  to EINVAL.  */
       if (save_errno == ENOTTY)
	 __set_errno (EINVAL);
       else
	 __set_errno (save_errno);

       return -1;
    }

  /* Make sure that we own the device.  */
  uid_t uid = __getuid ();
  if (st.st_uid != uid)
    {
      if (__chown (buf, uid, st.st_gid) < 0)
	goto helper;
    }

  static int tty_gid = -1;
  if (__builtin_expect (tty_gid == -1, 0))
    {
      char *grtmpbuf;
      struct group grbuf;
      size_t grbuflen = __sysconf (_SC_GETGR_R_SIZE_MAX);
      struct group *p;

      /* Get the group ID of the special `tty' group.  */
      if (grbuflen == (size_t) -1L)
	/* `sysconf' does not support _SC_GETGR_R_SIZE_MAX.
	   Try a moderate value.  */
	grbuflen = 1024;
      grtmpbuf = (char *) __alloca (grbuflen);
      __getgrnam_r (TTY_GROUP, &grbuf, grtmpbuf, grbuflen, &p);
      if (p != NULL)
	tty_gid = p->gr_gid;
    }
  gid_t gid = tty_gid == -1 ? __getgid () : tty_gid;

  /* Make sure the group of the device is that special group.  */
  if (st.st_gid != gid)
    {
      if (__chown (buf, uid, gid) < 0)
	goto helper;
    }

  /* Make sure the permission mode is set to readable and writable by
     the owner, and writable by the group.  */
  if ((st.st_mode & ACCESSPERMS) != (S_IRUSR|S_IWUSR|S_IWGRP))
    {
      if (__chmod (buf, S_IRUSR|S_IWUSR|S_IWGRP) < 0)
	goto helper;
    }

  retval = 0;
  goto cleanup;

  /* We have to use the helper program if it is available.  */
 helper:;

#ifdef HAVE_PT_CHOWN
  pid_t pid = __fork ();
  if (pid == -1)
    goto cleanup;
  else if (pid == 0)
    {
      /* Disable core dumps.  */
      struct rlimit rl = { 0, 0 };
      __setrlimit (RLIMIT_CORE, &rl);

      /* We pass the master pseudo terminal as file descriptor PTY_FILENO.  */
      if (fd != PTY_FILENO)
	if (__dup2 (fd, PTY_FILENO) < 0)
	  _exit (FAIL_EBADF);

# ifdef CLOSE_ALL_FDS
      CLOSE_ALL_FDS ();
# endif

      execle (_PATH_PT_CHOWN, basename (_PATH_PT_CHOWN), NULL, NULL);
      _exit (FAIL_EXEC);
    }
  else
    {
      int w;

      if (__waitpid (pid, &w, 0) == -1)
	goto cleanup;
      if (!WIFEXITED (w))
	__set_errno (ENOEXEC);
      else
	switch (WEXITSTATUS (w))
	  {
	  case 0:
	    retval = 0;
	    break;
	  case FAIL_EBADF:
	    __set_errno (EBADF);
	    break;
	  case FAIL_EINVAL:
	    __set_errno (EINVAL);
	    break;
	  case FAIL_EACCES:
	    __set_errno (EACCES);
	    break;
	  case FAIL_EXEC:
	    __set_errno (ENOEXEC);
	    break;
	  case FAIL_ENOMEM:
	    __set_errno (ENOMEM);
	    break;

	  default:
	    assert(! "getpt: internal error: invalid exit code from pt_chown");
	  }
    }
#endif

 cleanup:
  if (buf != _buf)
    free (buf);

  return retval;
}

CVE-2013-2207, BZ #15755: Disable pt_chown.

The helper binary pt_chown tricked into granting access to another
user's pseudo-terminal.

Pre-conditions for the attack:

 * Attacker with local user account
 * Kernel with FUSE support
 * "user_allow_other" in /etc/fuse.conf
 * Victim with allocated slave in /dev/pts

Using the setuid installed pt_chown and a weak check on whether a file
descriptor is a tty, an attacker could fake a pty check using FUSE and
trick pt_chown to grant ownership of a pty descriptor that the current
user does not own.  It cannot access /dev/pts/ptmx however.

In most modern distributions pt_chown is not needed because devpts
is enabled by default. The fix for this CVE is to disable building
and using pt_chown by default. We still provide a configure option
to enable hte use of pt_chown but distributions do so at their own
risk.


pts_name (int fd, char **pts, size_t buf_len, struct stat64 *stp)
{
  int rv;
  char *buf = *pts;

  for (;;)
    {
      char *new_buf;

      if (buf_len)
	{
	  rv = __ptsname_internal (fd, buf, buf_len, stp);
	  if (rv != 0)
	    {
	      if (rv == ENOTTY)
		/* ptsname_r returns with ENOTTY to indicate
		   a descriptor not referring to a pty master.
		   For this condition, grantpt must return EINVAL.  */
		rv = EINVAL;
	      errno = rv;	/* Not necessarily set by __ptsname_r.  */
	      break;
	    }

	  if (memchr (buf, '\0', buf_len))
	    /* We succeeded and the returned name fit in the buffer.  */
	    break;

	  /* Try again with a longer buffer.  */
	  buf_len += buf_len;	/* Double it */
	}
      else
	/* No initial buffer; start out by mallocing one.  */
	buf_len = 128;		/* First time guess.  */

      if (buf != *pts)
	/* We've already malloced another buffer at least once.  */
	new_buf = (char *) realloc (buf, buf_len);
      else
	new_buf = (char *) malloc (buf_len);
      if (! new_buf)
	{
	  rv = -1;
	  __set_errno (ENOMEM);
	  break;
	}
      buf = new_buf;
    }

  if (rv == 0)
    *pts = buf;		/* Return buffer to the user.  */
  else if (buf != *pts)
    free (buf);		/* Free what we malloced when returning an error.  */

  return rv;
}