static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field)
{
    switch (PB_HTYPE(field->type))
    {
        case PB_HTYPE_REQUIRED:
            return decode_basic_field(stream, wire_type, field);

        case PB_HTYPE_OPTIONAL:
            if (field->pSize != NULL)
                *(bool*)field->pSize = true;
            return decode_basic_field(stream, wire_type, field);

        case PB_HTYPE_REPEATED:
            if (wire_type == PB_WT_STRING
                && PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE)
            {
                /* Packed array */
                bool status = true;
                pb_istream_t substream;
                pb_size_t *size = (pb_size_t*)field->pSize;
                field->pData = (char*)field->pField + field->data_size * (*size);

                if (!pb_make_string_substream(stream, &substream))
                    return false;

                while (substream.bytes_left > 0 && *size < field->array_size)
                {
                    if (!decode_basic_field(&substream, PB_WT_PACKED, field))
                    {
                        status = false;
                        break;
                    }
                    (*size)++;
                    field->pData = (char*)field->pData + field->data_size;
                }

                if (substream.bytes_left != 0)
                    PB_RETURN_ERROR(stream, "array overflow");
                if (!pb_close_string_substream(stream, &substream))
                    return false;

                return status;
            }
            else
            {
                /* Repeated field */
                pb_size_t *size = (pb_size_t*)field->pSize;
                field->pData = (char*)field->pField + field->data_size * (*size);

                if ((*size)++ >= field->array_size)
                    PB_RETURN_ERROR(stream, "array overflow");

                return decode_basic_field(stream, wire_type, field);
            }

        case PB_HTYPE_ONEOF:
            if (PB_LTYPE_IS_SUBMSG(field->type) &&
                *(pb_size_t*)field->pSize != field->tag)
            {
                /* We memset to zero so that any callbacks are set to NULL.
                 * This is because the callbacks might otherwise have values
                 * from some other union field.
                 * If callbacks are needed inside oneof field, use .proto
                 * option submsg_callback to have a separate callback function
                 * that can set the fields before submessage is decoded.
                 * pb_dec_submessage() will set any default values. */
                memset(field->pData, 0, (size_t)field->data_size);

                /* Set default values for the submessage fields. */
                if (field->submsg_desc->default_value != NULL ||
                    field->submsg_desc->field_callback != NULL ||
                    field->submsg_desc->submsg_info[0] != NULL)
                {
                    pb_field_iter_t submsg_iter;
                    if (pb_field_iter_begin(&submsg_iter, field->submsg_desc, field->pData))
                    {
                        if (!pb_message_set_to_defaults(&submsg_iter))
                            PB_RETURN_ERROR(stream, "failed to set defaults");
                    }
                }
            }
            *(pb_size_t*)field->pSize = field->tag;

            return decode_basic_field(stream, wire_type, field);

        default:
            PB_RETURN_ERROR(stream, "invalid field type");
    }
}

Fix invalid free() with oneof (#647)

Nanopb would call free() or realloc() on an invalid
(attacker controlled) pointer value when all the following
conditions are true:

- PB_ENABLE_MALLOC is defined at the compile time
- Message definition contains an oneof field, and the oneof
  contains at least one pointer type field and at least one
  non-pointer type field.
- Data being decoded first contains a non-pointer value for
  the oneof field, and later contains an overwriting pointer
  value.

Depending on message layout, the bug may not be exploitable in all
cases, but it is known to be exploitable at least with string and
bytes fields. Actual security impact will also depend on the heap
implementation used.

#define checkreturn

#define checkreturn __attribute__((warn_unused_result))

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);

#define PB_WT_PACKED ((pb_wire_type_t)0xFF)


bool pb_decode_double_as_float(pb_istream_t *stream, float *dest)
{
    uint_least8_t sign;
    int exponent;
    uint32_t mantissa;
    uint64_t value;
    union { float f; uint32_t i; } out;

    if (!pb_decode_fixed64(stream, &value))
        return false;

    /* Decompose input value */
    sign = (uint_least8_t)((value >> 63) & 1);
    exponent = (int)((value >> 52) & 0x7FF) - 1023;
    mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */

    /* Figure if value is in range representable by floats. */
    if (exponent == 1024)
    {
        /* Special value */
        exponent = 128;
        mantissa >>= 1;
    }
    else
    {
        if (exponent > 127)
        {
            /* Too large, convert to infinity */
            exponent = 128;
            mantissa = 0;
        }
        else if (exponent < -150)
        {
            /* Too small, convert to zero */
            exponent = -127;
            mantissa = 0;
        }
        else if (exponent < -126)
        {
            /* Denormalized */
            mantissa |= 0x1000000;
            mantissa >>= (-126 - exponent);
            exponent = -127;
        }

        /* Round off mantissa */
        mantissa = (mantissa + 1) >> 1;

        /* Check if mantissa went over 2.0 */
        if (mantissa & 0x800000)
        {
            exponent += 1;
            mantissa &= 0x7FFFFF;
            mantissa >>= 1;
        }
    }

    /* Combine fields */
    out.i = mantissa;
    out.i |= (uint32_t)(exponent + 127) << 23;
    out.i |= (uint32_t)sign << 31;

    *dest = out.f;
    return true;
}

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

#define pb_uint64_t uint32_t

#define pb_uint64_t uint64_t

typedef struct {
    uint32_t bitfield[(PB_MAX_REQUIRED_FIELDS + 31) / 32];
} pb_fields_seen_t;


static bool pb_field_set_to_default(pb_field_iter_t *field)
{
    pb_type_t type;
    type = field->type;

    if (PB_LTYPE(type) == PB_LTYPE_EXTENSION)
    {
        pb_extension_t *ext = *(pb_extension_t* const *)field->pData;
        while (ext != NULL)
        {
            pb_field_iter_t ext_iter;
            if (pb_field_iter_begin_extension(&ext_iter, ext))
            {
                ext->found = false;
                if (!pb_message_set_to_defaults(&ext_iter))
                    return false;
            }
            ext = ext->next;
        }
    }
    else if (PB_ATYPE(type) == PB_ATYPE_STATIC)
    {
        bool init_data = true;
        if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && field->pSize != NULL)
        {
            /* Set has_field to false. Still initialize the optional field
             * itself also. */
            *(bool*)field->pSize = false;
        }
        else if (PB_HTYPE(type) == PB_HTYPE_REPEATED ||
                 PB_HTYPE(type) == PB_HTYPE_ONEOF)
        {
            /* REPEATED: Set array count to 0, no need to initialize contents.
               ONEOF: Set which_field to 0. */
            *(pb_size_t*)field->pSize = 0;
            init_data = false;
        }

        if (init_data)
        {
            if (PB_LTYPE_IS_SUBMSG(field->type) &&
                (field->submsg_desc->default_value != NULL ||
                 field->submsg_desc->field_callback != NULL ||
                 field->submsg_desc->submsg_info[0] != NULL))
            {
                /* Initialize submessage to defaults.
                 * Only needed if it has default values
                 * or callback/submessage fields. */
                pb_field_iter_t submsg_iter;
                if (pb_field_iter_begin(&submsg_iter, field->submsg_desc, field->pData))
                {
                    if (!pb_message_set_to_defaults(&submsg_iter))
                        return false;
                }
            }
            else
            {
                /* Initialize to zeros */
                memset(field->pData, 0, (size_t)field->data_size);
            }
        }
    }
    else if (PB_ATYPE(type) == PB_ATYPE_POINTER)
    {
        /* Initialize the pointer to NULL. */
        *(void**)field->pField = NULL;

        /* Initialize array count to 0. */
        if (PB_HTYPE(type) == PB_HTYPE_REPEATED ||
            PB_HTYPE(type) == PB_HTYPE_ONEOF)
        {
            *(pb_size_t*)field->pSize = 0;
        }
    }
    else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK)
    {
        /* Don't overwrite callback */
    }

    return true;
}

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);


static bool checkreturn decode_extension(pb_istream_t *stream,
    uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension)
{
    size_t pos = stream->bytes_left;

    while (extension != NULL && pos == stream->bytes_left)
    {
        bool status;
        if (extension->type->decode)
            status = extension->type->decode(stream, extension, tag, wire_type);
        else
            status = default_extension_decoder(stream, extension, tag, wire_type);

        if (!status)
            return false;

        extension = extension->next;
    }

    return true;
}

#define checkreturn

#define checkreturn __attribute__((warn_unused_result))

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);

#define pb_uint64_t uint32_t

#define PB_WT_PACKED ((pb_wire_type_t)0xFF)

typedef struct {
    uint32_t bitfield[(PB_MAX_REQUIRED_FIELDS + 31) / 32];
} pb_fields_seen_t;


static bool checkreturn pb_decode_inner(pb_istream_t *stream, const pb_msgdesc_t *fields, void *dest_struct, unsigned int flags)
{
    uint32_t extension_range_start = 0;
    pb_extension_t *extensions = NULL;

    /* 'fixed_count_field' and 'fixed_count_size' track position of a repeated fixed
     * count field. This can only handle _one_ repeated fixed count field that
     * is unpacked and unordered among other (non repeated fixed count) fields.
     */
    pb_size_t fixed_count_field = PB_SIZE_MAX;
    pb_size_t fixed_count_size = 0;
    pb_size_t fixed_count_total_size = 0;

    pb_fields_seen_t fields_seen = {{0, 0}};
    const uint32_t allbits = ~(uint32_t)0;
    pb_field_iter_t iter;

    if (pb_field_iter_begin(&iter, fields, dest_struct))
    {
        if ((flags & PB_DECODE_NOINIT) == 0)
        {
            if (!pb_message_set_to_defaults(&iter))
                PB_RETURN_ERROR(stream, "failed to set defaults");
        }
    }

    while (stream->bytes_left)
    {
        uint32_t tag;
        pb_wire_type_t wire_type;
        bool eof;

        if (!pb_decode_tag(stream, &wire_type, &tag, &eof))
        {
            if (eof)
                break;
            else
                return false;
        }

        if (tag == 0)
        {
          if (flags & PB_DECODE_NULLTERMINATED)
          {
            break;
          }
          else
          {
            PB_RETURN_ERROR(stream, "zero tag");
          }
        }

        if (!pb_field_iter_find(&iter, tag) || PB_LTYPE(iter.type) == PB_LTYPE_EXTENSION)
        {
            /* No match found, check if it matches an extension. */
            if (extension_range_start == 0)
            {
                if (pb_field_iter_find_extension(&iter))
                {
                    extensions = *(pb_extension_t* const *)iter.pData;
                    extension_range_start = iter.tag;
                }

                if (!extensions)
                {
                    extension_range_start = (uint32_t)-1;
                }
            }

            if (tag >= extension_range_start)
            {
                size_t pos = stream->bytes_left;

                if (!decode_extension(stream, tag, wire_type, extensions))
                    return false;

                if (pos != stream->bytes_left)
                {
                    /* The field was handled */
                    continue;
                }
            }

            /* No match found, skip data */
            if (!pb_skip_field(stream, wire_type))
                return false;
            continue;
        }

        /* If a repeated fixed count field was found, get size from
         * 'fixed_count_field' as there is no counter contained in the struct.
         */
        if (PB_HTYPE(iter.type) == PB_HTYPE_REPEATED && iter.pSize == &iter.array_size)
        {
            if (fixed_count_field != iter.index) {
                /* If the new fixed count field does not match the previous one,
                 * check that the previous one is NULL or that it finished
                 * receiving all the expected data.
                 */
                if (fixed_count_field != PB_SIZE_MAX &&
                    fixed_count_size != fixed_count_total_size)
                {
                    PB_RETURN_ERROR(stream, "wrong size for fixed count field");
                }

                fixed_count_field = iter.index;
                fixed_count_size = 0;
                fixed_count_total_size = iter.array_size;
            }

            iter.pSize = &fixed_count_size;
        }

        if (PB_HTYPE(iter.type) == PB_HTYPE_REQUIRED
            && iter.required_field_index < PB_MAX_REQUIRED_FIELDS)
        {
            uint32_t tmp = ((uint32_t)1 << (iter.required_field_index & 31));
            fields_seen.bitfield[iter.required_field_index >> 5] |= tmp;
        }

        if (!decode_field(stream, wire_type, &iter))
            return false;
    }

    /* Check that all elements of the last decoded fixed count field were present. */
    if (fixed_count_field != PB_SIZE_MAX &&
        fixed_count_size != fixed_count_total_size)
    {
        PB_RETURN_ERROR(stream, "wrong size for fixed count field");
    }

    /* Check that all required fields were present. */
    {
        pb_size_t req_field_count = iter.descriptor->required_field_count;

        if (req_field_count > 0)
        {
            pb_size_t i;

            if (req_field_count > PB_MAX_REQUIRED_FIELDS)
                req_field_count = PB_MAX_REQUIRED_FIELDS;

            /* Check the whole words */
            for (i = 0; i < (req_field_count >> 5); i++)
            {
                if (fields_seen.bitfield[i] != allbits)
                    PB_RETURN_ERROR(stream, "missing required field");
            }

            /* Check the remaining bits (if any) */
            if ((req_field_count & 31) != 0)
            {
                if (fields_seen.bitfield[req_field_count >> 5] !=
                    (allbits >> (uint_least8_t)(32 - (req_field_count & 31))))
                {
                    PB_RETURN_ERROR(stream, "missing required field");
                }
            }
        }
    }

    return true;
}


pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t msglen)
{
    pb_istream_t stream;
    /* Cast away the const from buf without a compiler error.  We are
     * careful to use it only in a const manner in the callbacks.
     */
    union {
        void *state;
        const void *c_state;
    } state;
#ifdef PB_BUFFER_ONLY
    stream.callback = NULL;
#else
    stream.callback = &buf_read;
#endif
    state.c_state = buf;
    stream.state = state.state;
    stream.bytes_left = msglen;
#ifndef PB_NO_ERRMSG
    stream.errmsg = NULL;
#endif
    return stream;
}

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);


static void pb_release_single_field(pb_field_iter_t *field)
{
    pb_type_t type;
    type = field->type;

    if (PB_HTYPE(type) == PB_HTYPE_ONEOF)
    {
        if (*(pb_size_t*)field->pSize != field->tag)
            return; /* This is not the current field in the union */
    }

    /* Release anything contained inside an extension or submsg.
     * This has to be done even if the submsg itself is statically
     * allocated. */
    if (PB_LTYPE(type) == PB_LTYPE_EXTENSION)
    {
        /* Release fields from all extensions in the linked list */
        pb_extension_t *ext = *(pb_extension_t**)field->pData;
        while (ext != NULL)
        {
            pb_field_iter_t ext_iter;
            if (pb_field_iter_begin_extension(&ext_iter, ext))
            {
                pb_release_single_field(&ext_iter);
            }
            ext = ext->next;
        }
    }
    else if (PB_LTYPE_IS_SUBMSG(type) && PB_ATYPE(type) != PB_ATYPE_CALLBACK)
    {
        /* Release fields in submessage or submsg array */
        pb_size_t count = 1;

        if (PB_ATYPE(type) == PB_ATYPE_POINTER)
        {
            field->pData = *(void**)field->pField;
        }
        else
        {
            field->pData = field->pField;
        }

        if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
        {
            count = *(pb_size_t*)field->pSize;

            if (PB_ATYPE(type) == PB_ATYPE_STATIC && count > field->array_size)
            {
                /* Protect against corrupted _count fields */
                count = field->array_size;
            }
        }

        if (field->pData)
        {
            for (; count > 0; count--)
            {
                pb_release(field->submsg_desc, field->pData);
                field->pData = (char*)field->pData + field->data_size;
            }
        }
    }

    if (PB_ATYPE(type) == PB_ATYPE_POINTER)
    {
        if (PB_HTYPE(type) == PB_HTYPE_REPEATED &&
            (PB_LTYPE(type) == PB_LTYPE_STRING ||
             PB_LTYPE(type) == PB_LTYPE_BYTES))
        {
            /* Release entries in repeated string or bytes array */
            void **pItem = *(void***)field->pField;
            pb_size_t count = *(pb_size_t*)field->pSize;
            for (; count > 0; count--)
            {
                pb_free(*pItem);
                *pItem++ = NULL;
            }
        }

        if (PB_HTYPE(type) == PB_HTYPE_REPEATED)
        {
            /* We are going to release the array, so set the size to 0 */
            *(pb_size_t*)field->pSize = 0;
        }

        /* Release main pointer */
        pb_free(*(void**)field->pField);
        *(void**)field->pField = NULL;
    }
}


static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, uint32_t *dest, bool *eof)
{
    pb_byte_t byte;
    uint32_t result;

    if (!pb_readbyte(stream, &byte))
    {
        if (stream->bytes_left == 0)
        {
            if (eof)
            {
                *eof = true;
            }
        }

        return false;
    }

    if ((byte & 0x80) == 0)
    {
        /* Quick case, 1 byte value */
        result = byte;
    }
    else
    {
        /* Multibyte case */
        uint_fast8_t bitpos = 7;
        result = byte & 0x7F;

        do
        {
            if (!pb_readbyte(stream, &byte))
                return false;

            if (bitpos >= 32)
            {
                /* Note: The varint could have trailing 0x80 bytes, or 0xFF for negative. */
                pb_byte_t sign_extension = (bitpos < 63) ? 0xFF : 0x01;
                bool valid_extension = ((byte & 0x7F) == 0x00 ||
                         ((result >> 31) != 0 && byte == sign_extension));

                if (bitpos >= 64 || !valid_extension)
                {
                    PB_RETURN_ERROR(stream, "varint overflow");
                }
            }
            else
            {
                result |= (uint32_t)(byte & 0x7F) << bitpos;
            }
            bitpos = (uint_fast8_t)(bitpos + 7);
        } while (byte & 0x80);

        if (bitpos == 35 && (byte & 0x70) != 0)
        {
            /* The last byte was at bitpos=28, so only bottom 4 bits fit. */
            PB_RETURN_ERROR(stream, "varint overflow");
        }
   }

   *dest = result;
   return true;
}

#define checkreturn

#define checkreturn __attribute__((warn_unused_result))

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);

#define pb_uint64_t uint32_t

typedef struct {
    uint32_t bitfield[(PB_MAX_REQUIRED_FIELDS + 31) / 32];
} pb_fields_seen_t;


static bool pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *field)
{
    pb_field_iter_t old_field = *field;
    pb_size_t old_tag = *(pb_size_t*)field->pSize; /* Previous which_ value */
    pb_size_t new_tag = field->tag; /* New which_ value */

    if (old_tag == 0)
        return true; /* Ok, no old data in union */

    if (old_tag == new_tag)
        return true; /* Ok, old data is of same type => merge */

    /* Release old data. The find can fail if the message struct contains
     * invalid data. */
    if (!pb_field_iter_find(&old_field, old_tag))
        PB_RETURN_ERROR(stream, "invalid union tag");

    pb_release_single_field(&old_field);

    if (PB_ATYPE(field->type) == PB_ATYPE_POINTER)
    {
        /* Initialize the pointer to NULL to make sure it is valid
         * even in case of error return. */
        *(void**)field->pField = NULL;
        field->pData = NULL;
    }

    return true;
}

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);

typedef struct {
    uint32_t bitfield[(PB_MAX_REQUIRED_FIELDS + 31) / 32];
} pb_fields_seen_t;


static bool checkreturn default_extension_decoder(pb_istream_t *stream,
    pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type)
{
    pb_field_iter_t iter;

    if (!pb_field_iter_begin_extension(&iter, extension))
        PB_RETURN_ERROR(stream, "invalid extension");

    if (iter.tag != tag || !iter.message)
        return true;

    extension->found = true;
    return decode_field(stream, wire_type, &iter);
}


static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field)
{
    bool status = true;
    bool submsg_consumed = false;
    pb_istream_t substream;

    if (!pb_make_string_substream(stream, &substream))
        return false;

    if (field->submsg_desc == NULL)
        PB_RETURN_ERROR(stream, "invalid field descriptor");

    /* Submessages can have a separate message-level callback that is called
     * before decoding the message. Typically it is used to set callback fields
     * inside oneofs. */
    if (PB_LTYPE(field->type) == PB_LTYPE_SUBMSG_W_CB && field->pSize != NULL)
    {
        /* Message callback is stored right before pSize. */
        pb_callback_t *callback = (pb_callback_t*)field->pSize - 1;
        if (callback->funcs.decode)
        {
            status = callback->funcs.decode(&substream, field, &callback->arg);

            if (substream.bytes_left == 0)
            {
                submsg_consumed = true;
            }
        }
    }

    /* Now decode the submessage contents */
    if (status && !submsg_consumed)
    {
        unsigned int flags = 0;

        /* Static required/optional fields are already initialized by top-level
         * pb_decode(), no need to initialize them again. */
        if (PB_ATYPE(field->type) == PB_ATYPE_STATIC &&
            PB_HTYPE(field->type) != PB_HTYPE_REPEATED)
        {
            flags = PB_DECODE_NOINIT;
        }

        status = pb_decode_inner(&substream, field->submsg_desc, field->pData, flags);
    }

    if (!pb_close_string_substream(stream, &substream))
        return false;

    return status;
}

#define checkreturn

#define checkreturn __attribute__((warn_unused_result))

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);


static void initialize_pointer_field(void *pItem, pb_field_iter_t *field)
{
    if (PB_LTYPE(field->type) == PB_LTYPE_STRING ||
        PB_LTYPE(field->type) == PB_LTYPE_BYTES)
    {
        *(void**)pItem = NULL;
    }
    else if (PB_LTYPE_IS_SUBMSG(field->type))
    {
        /* We memset to zero so that any callbacks are set to NULL.
         * Default values will be set by pb_dec_submessage(). */
        memset(pItem, 0, field->data_size);
    }
}


static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field)
{
#ifndef PB_ENABLE_MALLOC
    PB_UNUSED(wire_type);
    PB_UNUSED(field);
    PB_RETURN_ERROR(stream, "no malloc support");
#else
    switch (PB_HTYPE(field->type))
    {
        case PB_HTYPE_REQUIRED:
        case PB_HTYPE_OPTIONAL:
        case PB_HTYPE_ONEOF:
            if (PB_LTYPE_IS_SUBMSG(field->type) && *(void**)field->pField != NULL)
            {
                /* Duplicate field, have to release the old allocation first. */
                /* FIXME: Does this work correctly for oneofs? */
                pb_release_single_field(field);
            }

            if (PB_HTYPE(field->type) == PB_HTYPE_ONEOF)
            {
                *(pb_size_t*)field->pSize = field->tag;
            }

            if (PB_LTYPE(field->type) == PB_LTYPE_STRING ||
                PB_LTYPE(field->type) == PB_LTYPE_BYTES)
            {
                /* pb_dec_string and pb_dec_bytes handle allocation themselves */
                field->pData = field->pField;
                return decode_basic_field(stream, wire_type, field);
            }
            else
            {
                if (!allocate_field(stream, field->pField, field->data_size, 1))
                    return false;

                field->pData = *(void**)field->pField;
                initialize_pointer_field(field->pData, field);
                return decode_basic_field(stream, wire_type, field);
            }

        case PB_HTYPE_REPEATED:
            if (wire_type == PB_WT_STRING
                && PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE)
            {
                /* Packed array, multiple items come in at once. */
                bool status = true;
                pb_size_t *size = (pb_size_t*)field->pSize;
                size_t allocated_size = *size;
                pb_istream_t substream;

                if (!pb_make_string_substream(stream, &substream))
                    return false;

                while (substream.bytes_left)
                {
                    if (*size == PB_SIZE_MAX)
                    {
#ifndef PB_NO_ERRMSG
                        stream->errmsg = "too many array entries";
#endif
                        status = false;
                        break;
                    }

                    if ((size_t)*size + 1 > allocated_size)
                    {
                        /* Allocate more storage. This tries to guess the
                         * number of remaining entries. Round the division
                         * upwards. */
                        size_t remain = (substream.bytes_left - 1) / field->data_size + 1;
                        if (remain < PB_SIZE_MAX - allocated_size)
                            allocated_size += remain;
                        else
                            allocated_size += 1;

                        if (!allocate_field(&substream, field->pField, field->data_size, allocated_size))
                        {
                            status = false;
                            break;
                        }
                    }

                    /* Decode the array entry */
                    field->pData = *(char**)field->pField + field->data_size * (*size);
                    initialize_pointer_field(field->pData, field);
                    if (!decode_basic_field(&substream, PB_WT_PACKED, field))
                    {
                        status = false;
                        break;
                    }

                    (*size)++;
                }
                if (!pb_close_string_substream(stream, &substream))
                    return false;

                return status;
            }
            else
            {
                /* Normal repeated field, i.e. only one item at a time. */
                pb_size_t *size = (pb_size_t*)field->pSize;

                if (*size == PB_SIZE_MAX)
                    PB_RETURN_ERROR(stream, "too many array entries");

                if (!allocate_field(stream, field->pField, field->data_size, (size_t)(*size + 1)))
                    return false;

                field->pData = *(char**)field->pField + field->data_size * (*size);
                (*size)++;
                initialize_pointer_field(field->pData, field);
                return decode_basic_field(stream, wire_type, field);
            }

        default:
            PB_RETURN_ERROR(stream, "invalid field type");
    }
#endif
}


static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field)
{
    switch (PB_LTYPE(field->type))
    {
        case PB_LTYPE_BOOL:
            if (wire_type != PB_WT_VARINT && wire_type != PB_WT_PACKED)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_bool(stream, field);

        case PB_LTYPE_VARINT:
        case PB_LTYPE_UVARINT:
        case PB_LTYPE_SVARINT:
            if (wire_type != PB_WT_VARINT && wire_type != PB_WT_PACKED)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_varint(stream, field);

        case PB_LTYPE_FIXED32:
            if (wire_type != PB_WT_32BIT && wire_type != PB_WT_PACKED)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_decode_fixed32(stream, field->pData);

        case PB_LTYPE_FIXED64:
            if (wire_type != PB_WT_64BIT && wire_type != PB_WT_PACKED)
                PB_RETURN_ERROR(stream, "wrong wire type");

#ifdef PB_CONVERT_DOUBLE_FLOAT
            if (field->data_size == sizeof(float))
            {
                return pb_decode_double_as_float(stream, (float*)field->pData);
            }
#endif

#ifdef PB_WITHOUT_64BIT
            PB_RETURN_ERROR(stream, "invalid data_size");
#else
            return pb_decode_fixed64(stream, field->pData);
#endif

        case PB_LTYPE_BYTES:
            if (wire_type != PB_WT_STRING)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_bytes(stream, field);

        case PB_LTYPE_STRING:
            if (wire_type != PB_WT_STRING)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_string(stream, field);

        case PB_LTYPE_SUBMESSAGE:
        case PB_LTYPE_SUBMSG_W_CB:
            if (wire_type != PB_WT_STRING)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_submessage(stream, field);

        case PB_LTYPE_FIXED_LENGTH_BYTES:
            if (wire_type != PB_WT_STRING)
                PB_RETURN_ERROR(stream, "wrong wire type");

            return pb_dec_fixed_length_bytes(stream, field);

        default:
            PB_RETURN_ERROR(stream, "invalid field type");
    }
}


static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field)
{
    if (PB_LTYPE(field->type) == PB_LTYPE_UVARINT)
    {
        pb_uint64_t value, clamped;
        if (!pb_decode_varint(stream, &value))
            return false;

        /* Cast to the proper field size, while checking for overflows */
        if (field->data_size == sizeof(pb_uint64_t))
            clamped = *(pb_uint64_t*)field->pData = value;
        else if (field->data_size == sizeof(uint32_t))
            clamped = *(uint32_t*)field->pData = (uint32_t)value;
        else if (field->data_size == sizeof(uint_least16_t))
            clamped = *(uint_least16_t*)field->pData = (uint_least16_t)value;
        else if (field->data_size == sizeof(uint_least8_t))
            clamped = *(uint_least8_t*)field->pData = (uint_least8_t)value;
        else
            PB_RETURN_ERROR(stream, "invalid data_size");

        if (clamped != value)
            PB_RETURN_ERROR(stream, "integer too large");

        return true;
    }
    else
    {
        pb_uint64_t value;
        pb_int64_t svalue;
        pb_int64_t clamped;

        if (PB_LTYPE(field->type) == PB_LTYPE_SVARINT)
        {
            if (!pb_decode_svarint(stream, &svalue))
                return false;
        }
        else
        {
            if (!pb_decode_varint(stream, &value))
                return false;

            /* See issue 97: Google's C++ protobuf allows negative varint values to
            * be cast as int32_t, instead of the int64_t that should be used when
            * encoding. Nanopb versions before 0.2.5 had a bug in encoding. In order to
            * not break decoding of such messages, we cast <=32 bit fields to
            * int32_t first to get the sign correct.
            */
            if (field->data_size == sizeof(pb_int64_t))
                svalue = (pb_int64_t)value;
            else
                svalue = (int32_t)value;
        }

        /* Cast to the proper field size, while checking for overflows */
        if (field->data_size == sizeof(pb_int64_t))
            clamped = *(pb_int64_t*)field->pData = svalue;
        else if (field->data_size == sizeof(int32_t))
            clamped = *(int32_t*)field->pData = (int32_t)svalue;
        else if (field->data_size == sizeof(int_least16_t))
            clamped = *(int_least16_t*)field->pData = (int_least16_t)svalue;
        else if (field->data_size == sizeof(int_least8_t))
            clamped = *(int_least8_t*)field->pData = (int_least8_t)svalue;
        else
            PB_RETURN_ERROR(stream, "invalid data_size");

        if (clamped != svalue)
            PB_RETURN_ERROR(stream, "integer too large");

        return true;
    }
}

#define checkreturn

#define checkreturn __attribute__((warn_unused_result))

static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count);

static bool checkreturn

static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size);

static bool checkreturn decode_basic_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *field);

static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type);

static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_extension_t *extension);

static bool pb_field_set_to_default(pb_field_iter_t *field);

static bool pb_message_set_to_defaults(pb_field_iter_t *iter);

static bool checkreturn pb_dec_bool(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_iter_t *field);

static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size);

static void initialize_pointer_field(void *pItem, pb_field_iter_t *field);

static void pb_release_single_field(pb_field_iter_t *field);

#define pb_int64_t int32_t

#define pb_uint64_t uint32_t

#define pb_int64_t int64_t

#define pb_uint64_t uint64_t

typedef struct {
    uint32_t bitfield[(PB_MAX_REQUIRED_FIELDS + 31) / 32];
} pb_fields_seen_t;


void sigproc(int sig) {

  static int called = 0;
  int thread_id;

  if(called) return; else called = 1;
  shutdown_app = 1;

  for(thread_id=0; thread_id<num_threads; thread_id++)
    breakPcapLoop(thread_id);
}

Added fix to avoid potential heap buffer overflow in H.323 dissector
Modified HTTP report information to make it closer to the HTTP field names

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

int nDPI_LogLevel = 0;

u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

static pcap_dumper_t *extcap_dumper = NULL;

static char extcap_buf[16384];

static char *extcap_capture_fifo    = NULL;

static u_int16_t extcap_packet_filter = (u_int16_t)-1;

static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS];

static int dpdk_port_id = 0, dpdk_run_capture = 1;

void test_lib();

extern void ndpi_report_payload_stats();

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);


static void help(u_int long_help) {
  printf("Welcome to nDPI %s\n\n", ndpi_revision());

  printf("ndpiReader "
#ifndef USE_DPDK
	 "-i <file|device> "
#endif
	 "[-f <filter>][-s <duration>][-m <duration>]\n"
	 "          [-p <protos>][-l <loops> [-q][-d][-J][-h][-e <len>][-t][-v <level>]\n"
	 "          [-n <threads>][-w <file>][-c <file>][-C <file>][-j <file>][-x <file>]\n"
	 "          [-T <num>][-U <num>]\n\n"
	 "Usage:\n"
	 "  -i <file.pcap|device>     | Specify a pcap file/playlist to read packets from or a\n"
	 "                            | device for live capture (comma-separated list)\n"
	 "  -f <BPF filter>           | Specify a BPF filter for filtering selected traffic\n"
	 "  -s <duration>             | Maximum capture duration in seconds (live traffic capture only)\n"
	 "  -m <duration>             | Split analysis duration in <duration> max seconds\n"
	 "  -p <file>.protos          | Specify a protocol file (eg. protos.txt)\n"
	 "  -l <num loops>            | Number of detection loops (test only)\n"
	 "  -n <num threads>          | Number of threads. Default: number of interfaces in -i.\n"
	 "                            | Ignored with pcap files.\n"
#ifdef linux
         "  -g <id:id...>             | Thread affinity mask (one core id per thread)\n"
#endif
	 "  -d                        | Disable protocol guess and use only DPI\n"
	 "  -e <len>                  | Min human readeable string match len. Default %u\n"
	 "  -q                        | Quiet mode\n"
	 "  -J                        | Display flow SPLT (sequence of packet length and time)\n"
	 "                            | and BD (byte distribution). See https://github.com/cisco/joy\n"
	 "  -t                        | Dissect GTP/TZSP tunnels\n"
	 "  -P <a>:<b>:<c>:<d>:<e>    | Enable payload analysis:\n"
	 "                            | <a> = min pattern len to search\n"
	 "                            | <b> = max pattern len to search\n"
	 "                            | <c> = max num packets per flow\n"
	 "                            | <d> = max packet payload dissection\n"
	 "                            | <d> = max num reported payloads\n"
	 "                            | Default: %u:%u:%u:%u:%u\n"
	 "  -r                        | Print nDPI version and git revision\n"
	 "  -c <path>                 | Load custom categories from the specified file\n"
	 "  -C <path>                 | Write output in CSV format on the specified file\n"
	 "  -w <path>                 | Write test output on the specified file. This is useful for\n"
	 "                            | testing purposes in order to compare results across runs\n"
	 "  -h                        | This help\n"
	 "  -v <1|2|3>                | Verbose 'unknown protocol' packet print.\n"
	 "                            | 1 = verbose\n"
	 "                            | 2 = very verbose\n"
	 "                            | 3 = port stats\n"
	 "  -V <1-4>                  | nDPI logging level\n"
	 "                            | 1 - trace, 2 - debug, 3 - full debug\n"
	 "                            | >3 - full debug + dbg_proto = all\n"
	 "  -T <num>                  | Max number of TCP processed packets before giving up [default: %u]\n"
	 "  -U <num>                  | Max number of UDP processed packets before giving up [default: %u]\n"
	 ,
	 human_readeable_string_len,
	 min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
	 max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts);

#ifndef WIN32
  printf("\nExcap (wireshark) options:\n"
	 "  --extcap-interfaces\n"
	 "  --extcap-version\n"
	 "  --extcap-dlts\n"
	 "  --extcap-interface <name>\n"
	 "  --extcap-config\n"
	 "  --capture\n"
	 "  --extcap-capture-filter\n"
	 "  --fifo <path to file or pipe>\n"
	 "  --debug\n"
	 "  --dbg-proto proto|num[,...]\n"
    );
#endif

  if(long_help) {
    NDPI_PROTOCOL_BITMASK all;

    printf("\n\nnDPI supported protocols:\n");
    printf("%3s %-22s %-8s %-12s %s\n", "Id", "Protocol", "Layer_4", "Breed", "Category");
    num_threads = 1;

    NDPI_BITMASK_SET_ALL(all);
    ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);

    ndpi_dump_protocols(ndpi_info_mod);
  }
  exit(!long_help);
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

u_int8_t verbose = 0, enable_joy_stats = 0;

u_int8_t human_readeable_string_len = 5;

u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int32_t max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads;

extern u_int16_t min_pattern_len, max_pattern_len;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}


static void deletePortsStats(struct port_stats *stats) {
  struct port_stats *current_port, *tmp;

  HASH_ITER(hh, stats, current_port, tmp) {
    HASH_DEL(stats, current_port);
    freeIpTree(current_port->addr_tree);
    free(current_port);
  }
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct receiver *receivers = NULL, *topReceivers = NULL;


static int getNextPcapFileFromPlaylist(u_int16_t thread_id, char filename[], u_int32_t filename_len) {

  if(playlist_fp[thread_id] == NULL) {
    if((playlist_fp[thread_id] = fopen(_pcap_file[thread_id], "r")) == NULL)
      return -1;
  }

next_line:
  if(fgets(filename, filename_len, playlist_fp[thread_id])) {
    int l = strlen(filename);
    if(filename[0] == '\0' || filename[0] == '#') goto next_line;
    if(filename[l-1] == '\n') filename[l-1] = '\0';
    return 0;
  } else {
    fclose(playlist_fp[thread_id]);
    playlist_fp[thread_id] = NULL;
    return -1;
  }
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

int nDPI_LogLevel = 0;

char *_debug_protocols = NULL;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int32_t max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads;

extern u_int16_t min_pattern_len, max_pattern_len;

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}


static int port_stats_sort(void *_a, void *_b) {
  struct port_stats *a = (struct port_stats*)_a;
  struct port_stats *b = (struct port_stats*)_b;

  if(b->num_pkts == 0 && a->num_pkts == 0)
    return(b->num_flows - a->num_flows);

  return(b->num_pkts - a->num_pkts);
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

int nDPI_LogLevel = 0;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct receiver *receivers = NULL, *topReceivers = NULL;


static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_file) {
  u_int snaplen = 1536;
  int promisc = 1;
  char pcap_error_buffer[PCAP_ERRBUF_SIZE];
  pcap_t * pcap_handle = NULL;

  /* trying to open a live interface */
#ifdef USE_DPDK
  struct rte_mempool *mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS,
							  MBUF_CACHE_SIZE, 0,
							  RTE_MBUF_DEFAULT_BUF_SIZE,
							  rte_socket_id());

  if(mbuf_pool == NULL)
    rte_exit(EXIT_FAILURE, "Cannot create mbuf pool: are hugepages ok?\n");

  if(dpdk_port_init(dpdk_port_id, mbuf_pool) != 0)
    rte_exit(EXIT_FAILURE, "DPDK: Cannot init port %u: please see README.dpdk\n", dpdk_port_id);
#else
  /* Trying to open the interface */
  if((pcap_handle = pcap_open_live((char*)pcap_file, snaplen,
				   promisc, 500, pcap_error_buffer)) == NULL) {
    capture_for = capture_until = 0;

    live_capture = 0;
    num_threads = 1; /* Open pcap files in single threads mode */

    /* Trying to open a pcap file */
    if((pcap_handle = pcap_open_offline((char*)pcap_file, pcap_error_buffer)) == NULL) {
      char filename[256] = { 0 };

      if(strstr((char*)pcap_file, (char*)".pcap"))
	printf("ERROR: could not open pcap file %s: %s\n", pcap_file, pcap_error_buffer);

      /* Trying to open as a playlist as last attempt */
      else if((getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) != 0)
	      || ((pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) == NULL)) {
        /* This probably was a bad interface name, printing a generic error */
        printf("ERROR: could not open %s: %s\n", filename, pcap_error_buffer);
        exit(-1);
      } else {
        if((!quiet_mode))
	  printf("Reading packets from playlist %s...\n", pcap_file);
      }
    } else {
      if((!quiet_mode))
	printf("Reading packets from pcap file %s...\n", pcap_file);
    }
  } else {
    live_capture = 1;

    if((!quiet_mode)) {
#ifdef USE_DPDK
      printf("Capturing from DPDK (port 0)...\n");
#else
      printf("Capturing live traffic from device %s...\n", pcap_file);
#endif
    }
  }

  configurePcapHandle(pcap_handle);
#endif /* !DPDK */

  if(capture_for > 0) {
    if((!quiet_mode))
      printf("Capturing traffic up to %u seconds\n", (unsigned int)capture_for);

#ifndef WIN32
    alarm(capture_for);
    signal(SIGALRM, sigproc);
#endif
  }

  return pcap_handle;
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

int nDPI_LogLevel = 0;

char *_debug_protocols = NULL;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int16_t min_pattern_len, max_pattern_len;

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct receiver *receivers = NULL, *topReceivers = NULL;


char* intoaV4(u_int32_t addr, char* buf, u_int16_t bufLen) {
  char *cp;
  int n;

  cp = &buf[bufLen];
  *--cp = '\0';

  n = 4;
  do {
    u_int byte = addr & 0xff;

    *--cp = byte % 10 + '0';
    byte /= 10;
    if(byte > 0) {
      *--cp = byte % 10 + '0';
      byte /= 10;
      if(byte > 0)
	*--cp = byte + '0';
    }
    if(n > 1)
      *--cp = '.';
    addr >>= 8;
  } while (--n > 0);

  return(cp);
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

int nDPI_LogLevel = 0;

char *_debug_protocols = NULL;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static int core_affinity[MAX_NUM_READER_THREADS];

static u_int32_t num_flows;

extern u_int32_t max_num_packets_per_flow, max_packet_payload_dissection, max_num_reported_top_payloads;

extern u_int16_t min_pattern_len, max_pattern_len;

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct ndpi_packet_trailer {
  u_int32_t magic; /* 0x19682017 */
  u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
  char name[16];
}

static char extcap_buf[16384];

static char *extcap_capture_fifo    = NULL;

static u_int16_t extcap_packet_filter = (u_int16_t)-1;

struct reader_thread {
  struct ndpi_workflow *workflow;
  pthread_t pthread;
  u_int64_t last_idle_scan_time;
  u_int32_t idle_scan_idx;
  u_int32_t num_idle_flows;
  struct ndpi_flow_info *idle_flows[IDLE_SCAN_BUDGET];
}

u_int32_t current_ndpi_memory = 0, max_ndpi_memory = 0;


void extcap_dlts() {
  u_int dlts_number = DLT_EN10MB;
  printf("dlt {number=%u}{name=%s}{display=%s}\n", dlts_number, "ndpi", "nDPI Interface");
  exit(0);
}

extern void ndpi_self_check_host_match();

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct ndpi_packet_trailer {
  u_int32_t magic; /* 0x19682017 */
  u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
  char name[16];
}

void test_lib();

extern void ndpi_report_payload_stats();

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);

static struct option longopts[] = {
  /* mandatory extcap options */
  { "extcap-interfaces", no_argument, NULL, '0'},
  { "extcap-version", optional_argument, NULL, '1'},
  { "extcap-dlts", no_argument, NULL, '2'},
  { "extcap-interface", required_argument, NULL, '3'},
  { "extcap-config", no_argument, NULL, '4'},
  { "capture", no_argument, NULL, '5'},
  { "extcap-capture-filter", required_argument, NULL, '6'},
  { "fifo", required_argument, NULL, '7'},
  { "debug", no_argument, NULL, '8'},
  { "dbg-proto", required_argument, NULL, 257},
  { "ndpi-proto-filter", required_argument, NULL, '9'},

  /* ndpiReader options */
  { "enable-protocol-guess", no_argument, NULL, 'd'},
  { "categories", required_argument, NULL, 'c'},
  { "csv-dump", required_argument, NULL, 'C'},
  { "interface", required_argument, NULL, 'i'},
  { "filter", required_argument, NULL, 'f'},
  { "cpu-bind", required_argument, NULL, 'g'},
  { "loops", required_argument, NULL, 'l'},
  { "num-threads", required_argument, NULL, 'n'},

  { "protos", required_argument, NULL, 'p'},
  { "capture-duration", required_argument, NULL, 's'},
  { "decode-tunnels", no_argument, NULL, 't'},
  { "revision", no_argument, NULL, 'r'},
  { "verbose", no_argument, NULL, 'v'},
  { "version", no_argument, NULL, 'V'},
  { "help", no_argument, NULL, 'h'},
  { "joy", required_argument, NULL, 'J'},
  { "payload-analysis", required_argument, NULL, 'P'},
  { "result-path", required_argument, NULL, 'w'},
  { "quiet", no_argument, NULL, 'q'},

  {0, 0, 0, 0}
};

struct ndpi_proto_sorter {
  int id;
  char name[16];
}


static void on_protocol_discovered(struct ndpi_workflow * workflow,
				   struct ndpi_flow_info * flow,
				   void * udata) {
  ;
}


void bpf_filter_port_array_init(int array[], int size) {
  int i;
  for(i=0; i<size; i++)
    array[i] = INIT_VAL;
}

int nDPI_LogLevel = 0;

u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;

static int core_affinity[MAX_NUM_READER_THREADS];

extern void ndpi_self_check_host_match();

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

static int dpdk_port_id = 0, dpdk_run_capture = 1;

void test_lib();

extern void ndpi_report_payload_stats();

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);

struct ndpi_proto_sorter {
  int id;
  char name[16];
}

struct timezone {
    int tz_minuteswest; /* minutes W of Greenwich */
    int tz_dsttime;     /* type of dst correction */
  }


void * processing_thread(void *_thread_id) {
  long thread_id = (long) _thread_id;
  char pcap_error_buffer[PCAP_ERRBUF_SIZE];

#if defined(linux) && defined(HAVE_PTHREAD_SETAFFINITY_NP)
  if(core_affinity[thread_id] >= 0) {
    cpu_set_t cpuset;

    CPU_ZERO(&cpuset);
    CPU_SET(core_affinity[thread_id], &cpuset);

    if(pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset) != 0)
      fprintf(stderr, "Error while binding thread %ld to core %d\n", thread_id, core_affinity[thread_id]);
    else {
      if((!quiet_mode)) printf("Running thread %ld on core %d...\n", thread_id, core_affinity[thread_id]);
    }
  } else
#endif
    if((!quiet_mode)) printf("Running thread %ld...\n", thread_id);

#ifdef USE_DPDK
  while(dpdk_run_capture) {
    struct rte_mbuf *bufs[BURST_SIZE];
    u_int16_t num = rte_eth_rx_burst(dpdk_port_id, 0, bufs, BURST_SIZE);
    u_int i;

    if(num == 0) {
      usleep(1);
      continue;
    }

    for(i = 0; i < PREFETCH_OFFSET && i < num; i++)
      rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void *));

    for(i = 0; i < num; i++) {
      char *data = rte_pktmbuf_mtod(bufs[i], char *);
      int len = rte_pktmbuf_pkt_len(bufs[i]);
      struct pcap_pkthdr h;

      h.len = h.caplen = len;
      gettimeofday(&h.ts, NULL);

      ndpi_process_packet((u_char*)&thread_id, &h, (const u_char *)data);
      rte_pktmbuf_free(bufs[i]);
    }
  }
#else
pcap_loop:
  runPcapLoop(thread_id);

  if(playlist_fp[thread_id] != NULL) { /* playlist: read next file */
    char filename[256];

    if(getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) == 0 &&
       (ndpi_thread_info[thread_id].workflow->pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) != NULL) {
      configurePcapHandle(ndpi_thread_info[thread_id].workflow->pcap_handle);
      goto pcap_loop;
    }
  }
#endif

  return NULL;
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

int nDPI_LogLevel = 0;

char *_debug_protocols = NULL;

u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int16_t min_pattern_len, max_pattern_len;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct receiver *receivers = NULL, *topReceivers = NULL;


static char* print_cipher(ndpi_cipher_weakness c) {
  switch(c) {
  case ndpi_cipher_insecure:
    return(" (INSECURE)");
    break;

  case ndpi_cipher_weak:
    return(" (WEAK)");
    break;

  default:
    return("");
  }
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

char *_debug_protocols = NULL;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct ndpi_packet_trailer {
  u_int32_t magic; /* 0x19682017 */
  u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
  char name[16];
}

static pcap_dumper_t *extcap_dumper = NULL;

static char extcap_buf[16384];

static char *extcap_capture_fifo    = NULL;

static u_int16_t extcap_packet_filter = (u_int16_t)-1;

static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS];

static int dpdk_port_id = 0, dpdk_run_capture = 1;

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);


static void node_print_unknown_proto_walker(const void *node,
					    ndpi_VISIT which, int depth, void *user_data) {
  struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node;
  u_int16_t thread_id = *((u_int16_t*)user_data);

  if((flow->detected_protocol.master_protocol != NDPI_PROTOCOL_UNKNOWN)
     || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN))
    return;

  if((which == ndpi_preorder) || (which == ndpi_leaf)) {
    /* Avoid walking the same node multiple times */
    all_flows[num_flows].thread_id = thread_id, all_flows[num_flows].flow = flow;
    num_flows++;
  }
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

int nDPI_LogLevel = 0;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int16_t min_pattern_len, max_pattern_len;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct port_flow_info {
  u_int32_t port; /* key */
  u_int32_t num_flows;
  UT_hash_handle hh;
}

struct single_flow_info {
  u_int32_t saddr; /* key */
  u_int8_t version; /* IP version */
  struct port_flow_info *ports;
  u_int32_t tot_flows;
  UT_hash_handle hh;
}

struct single_flow_info *scannerHosts = NULL;

struct receiver {
  u_int32_t addr; /* key */
  u_int8_t version; /* IP version */
  u_int32_t num_pkts;
  UT_hash_handle hh;
}

struct receiver *receivers = NULL, *topReceivers = NULL;


static void deleteScanners(struct single_flow_info *scanners) {
  struct single_flow_info *s, *tmp;
  struct port_flow_info *p, *tmp2;

  HASH_ITER(hh, scanners, s, tmp) {
    HASH_ITER(hh, s->ports, p, tmp2) {
      if(s->ports) HASH_DEL(s->ports, p);
      free(p);
    }
    HASH_DEL(scanners, s);
    free(s);
  }
}


static void breakPcapLoop(u_int16_t thread_id) {
#ifdef USE_DPDK
  dpdk_run_capture = 0;
#else
  if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL) {
    pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
  }
#endif
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

static u_int8_t live_capture = 0;

static u_int8_t undetected_flows_deleted = 0;

char *_debug_protocols = NULL;

static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

static u_int16_t decode_tunnels = 0;

static u_int16_t num_loops = 1;

static u_int8_t shutdown_app = 0, quiet_mode = 0;

static u_int8_t num_threads = 1;

static struct timeval startup_time, begin, end;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct timeval pcap_start = { 0, 0}, pcap_end = { 0, 0 };

static time_t capture_for = 0;

static time_t capture_until = 0;

static u_int32_t num_flows;

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern u_int16_t min_pattern_len, max_pattern_len;

extern void ndpi_self_check_host_match();

struct flow_info {
  struct ndpi_flow_info *flow;
  u_int16_t thread_id;
}

static struct flow_info *all_flows;

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct single_flow_info *scannerHosts = NULL;

struct receiver *receivers = NULL, *topReceivers = NULL;

struct ndpi_packet_trailer {
  u_int32_t magic; /* 0x19682017 */
  u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
  char name[16];
}

static pcap_dumper_t *extcap_dumper = NULL;

static char extcap_buf[16384];

static char *extcap_capture_fifo    = NULL;

static u_int16_t extcap_packet_filter = (u_int16_t)-1;

struct reader_thread {
  struct ndpi_workflow *workflow;
  pthread_t pthread;
  u_int64_t last_idle_scan_time;
  u_int32_t idle_scan_idx;
  u_int32_t num_idle_flows;
  struct ndpi_flow_info *idle_flows[IDLE_SCAN_BUDGET];
}

static struct reader_thread ndpi_thread_info[MAX_NUM_READER_THREADS];

static int dpdk_port_id = 0, dpdk_run_capture = 1;

void test_lib();

extern void ndpi_report_payload_stats();

FILE *trace = NULL;

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);


static void runPcapLoop(u_int16_t thread_id) {
  if((!shutdown_app) && (ndpi_thread_info[thread_id].workflow->pcap_handle != NULL))
    if(pcap_loop(ndpi_thread_info[thread_id].workflow->pcap_handle, -1, &ndpi_process_packet, (u_char*)&thread_id) < 0)
      printf("Error while reading pcap file: '%s'\n", pcap_geterr(ndpi_thread_info[thread_id].workflow->pcap_handle));
}


void bpf_filter_host_array_add(const char *filter_array[48], int size, const char *host) {
  int i;
  int r;
  for(i=0; i<size; i++) {
    if((filter_array[i] != NULL) && (r = strcmp(filter_array[i], host)) == 0)
      return;
    if(filter_array[i] == NULL) {
      filter_array[i] = host;
      return;
    }
  }
  fprintf(stderr,"bpf_filter_host_array_add: max array size is reached!\n");
  exit(-1);
}

static char *_pcap_file[MAX_NUM_READER_THREADS];

static FILE *playlist_fp[MAX_NUM_READER_THREADS] = { NULL };

static FILE *results_file           = NULL;

static char *results_path           = NULL;

static char * bpfFilter             = NULL;

static char *_protoFilePath         = NULL;

static char *_customCategoryFilePath= NULL;

static FILE *csv_fp                 = NULL;

int nDPI_LogLevel = 0;

char *_debug_protocols = NULL;

u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;

static int core_affinity[MAX_NUM_READER_THREADS];

static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;

extern void ndpi_self_check_host_match();

struct info_pair {
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
}

typedef struct node_a{
  u_int32_t addr;
  u_int8_t version; /* IP version */
  char proto[16]; /*app level protocol*/
  int count;
  struct node_a *left, *right;
}addr_node;

struct port_stats {
  u_int32_t port; /* we'll use this field as the key */
  u_int32_t num_pkts, num_bytes;
  u_int32_t num_flows;
  u_int32_t num_addr; /*number of distinct IP addresses */
  u_int32_t cumulative_addr; /*cumulative some of IP addresses */
  addr_node *addr_tree; /* tree of distinct IP addresses */
  struct info_pair top_ip_addrs[MAX_NUM_IP_ADDRESS];
  u_int8_t hasTopHost; /* as boolean flag */
  u_int32_t top_host;  /* host that is contributed to > 95% of traffic */
  u_int8_t version;    /* top host's ip version */
  char proto[16];      /* application level protocol of top host */
  UT_hash_handle hh;   /* makes this structure hashable */
}

struct port_stats *srcStats = NULL, *dstStats = NULL;

struct single_flow_info *scannerHosts = NULL;

struct receiver *receivers = NULL, *topReceivers = NULL;

struct ndpi_packet_trailer {
  u_int32_t magic; /* 0x19682017 */
  u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
  char name[16];
}

static pcap_dumper_t *extcap_dumper = NULL;

static char extcap_buf[16384];

static char *extcap_capture_fifo    = NULL;

static int dpdk_port_id = 0, dpdk_run_capture = 1;

void test_lib();

extern void ndpi_report_payload_stats();

FILE *trace = NULL;

static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle);