MagickExport void ConvertHCLToRGB(const double hue,const double chroma,
  const double luma,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    b,
    c,
    g,
    h,
    m,
    r,
    x;

  /*
    Convert HCL to RGB colorspace.
  */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  h=6.0*hue;
  c=chroma;
  x=c*(1.0-fabs(fmod(h,2.0)-1.0));
  r=0.0;
  g=0.0;
  b=0.0;
  if ((0.0 <= h) && (h < 1.0))
    {
      r=c;
      g=x;
    }
  else
    if ((1.0 <= h) && (h < 2.0))
      {
        r=x;
        g=c;
      }
    else
      if ((2.0 <= h) && (h < 3.0))
        {
          g=c;
          b=x;
        }
      else
        if ((3.0 <= h) && (h < 4.0))
          {
            g=x;
            b=c;
          }
        else
          if ((4.0 <= h) && (h < 5.0))
            {
              r=x;
              b=c;
            }
          else
            if ((5.0 <= h) && (h < 6.0))
              {
                r=c;
                b=x;
              }
  m=luma-(0.298839*r+0.586811*g+0.114350*b);
  *red=ClampToQuantum(QuantumRange*(r+m));
  *green=ClampToQuantum(QuantumRange*(g+m));
  *blue=ClampToQuantum(QuantumRange*(b+m));
}


MagickExport void ConvertRGBToLCHuv(const Quantum red,const Quantum green,
  const Quantum blue,double *luma,double *chroma,double *hue)
{
  double
    X,
    Y,
    Z;

  /*
    Convert RGB to LCHuv colorspace.
  */
  assert(luma != (double *) NULL);
  assert(chroma != (double *) NULL);
  assert(hue != (double *) NULL);
  ConvertRGBToXYZ(red,green,blue,&X,&Y,&Z);
  ConvertXYZToLCHuv(X,Y,Z,luma,chroma,hue);
}

/* No explicit global definitions found. */


MagickExport void ConvertHSIToRGB(const double hue,const double saturation,
  const double intensity,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    b,
    g,
    h,
    r;

  /*
    Convert HSI to RGB colorspace.
  */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  h=360.0*hue;
  h-=360.0*floor(h/360.0);
  if (h < 120.0)
    {
      b=intensity*(1.0-saturation);
      r=intensity*(1.0+saturation*cos(h*(MagickPI/180.0))/cos((60.0-h)*
        (MagickPI/180.0)));
      g=3.0*intensity-r-b;
    }
  else
    if (h < 240.0)
      {
        h-=120.0;
        r=intensity*(1.0-saturation);
        g=intensity*(1.0+saturation*cos(h*(MagickPI/180.0))/cos((60.0-h)*
          (MagickPI/180.0)));
        b=3.0*intensity-r-g;
      }
    else
      {
        h-=240.0;
        g=intensity*(1.0-saturation);
        b=intensity*(1.0+saturation*cos(h*(MagickPI/180.0))/cos((60.0-h)*
          (MagickPI/180.0)));
        r=3.0*intensity-g-b;
      }
  *red=ClampToQuantum(QuantumRange*r);
  *green=ClampToQuantum(QuantumRange*g);
  *blue=ClampToQuantum(QuantumRange*b);
}


MagickExport void ConvertHCLpToRGB(const double hue,const double chroma,
  const double luma,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    b,
    c,
    g,
    h,
    m,
    r,
    x,
    z;

  /*
    Convert HCLp to RGB colorspace.
  */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  h=6.0*hue;
  c=chroma;
  x=c*(1.0-fabs(fmod(h,2.0)-1.0));
  r=0.0;
  g=0.0;
  b=0.0;
  if ((0.0 <= h) && (h < 1.0))
    {
      r=c;
      g=x;
    }
  else
    if ((1.0 <= h) && (h < 2.0))
      {
        r=x;
        g=c;
      }
    else
      if ((2.0 <= h) && (h < 3.0))
        {
          g=c;
          b=x;
        }
      else
        if ((3.0 <= h) && (h < 4.0))
          {
            g=x;
            b=c;
          }
        else
          if ((4.0 <= h) && (h < 5.0))
            {
              r=x;
              b=c;
            }
          else
            if ((5.0 <= h) && (h < 6.0))
              {
                r=c;
                b=x;
              }
  m=luma-(0.298839*r+0.586811*g+0.114350*b);
  z=1.0;
  if (m < 0.0)
    {
      z=luma/(luma-m);
      m=0.0;
    }
  else
    if (m+c > 1.0)
      {
        z=(1.0-luma)/(m+c-luma);
        m=1.0-z*c;
      }
  *red=ClampToQuantum(QuantumRange*(z*r+m));
  *green=ClampToQuantum(QuantumRange*(z*g+m));
  *blue=ClampToQuantum(QuantumRange*(z*b+m));
}


MagickExport void ConvertLCHuvToRGB(const double luma,const double chroma,
  const double hue,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    X,
    Y,
    Z;

  /*
   Convert LCHuv to RGB colorspace.
 */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  ConvertLCHuvToXYZ(100.0*luma,255.0*(chroma-0.5),360.0*hue,&X,&Y,&Z);
  ConvertXYZToRGB(X,Y,Z,red,green,blue);
}


MagickExport void ConvertHWBToRGB(const double hue,const double whiteness,
  const double blackness,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    b,
    f,
    g,
    n,
    r,
    v;

  ssize_t
    i;

  /*
    Convert HWB to RGB colorspace.
  */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  v=1.0-blackness;
  if (fabs(hue-(-1.0)) < MagickEpsilon)
    {
      *red=ClampToQuantum(QuantumRange*v);
      *green=ClampToQuantum(QuantumRange*v);
      *blue=ClampToQuantum(QuantumRange*v);
      return;
    }
  i=CastDoubleToLong(floor(6.0*hue));
  f=6.0*hue-i;
  if ((i & 0x01) != 0)
    f=1.0-f;
  n=whiteness+f*(v-whiteness);  /* linear interpolation */
  switch (i)
  {
    default:
    case 6:
    case 0: r=v; g=n; b=whiteness; break;
    case 1: r=n; g=v; b=whiteness; break;
    case 2: r=whiteness; g=v; b=n; break;
    case 3: r=whiteness; g=n; b=v; break;
    case 4: r=n; g=whiteness; b=v; break;
    case 5: r=v; g=whiteness; b=n; break;
  }
  *red=ClampToQuantum(QuantumRange*r);
  *green=ClampToQuantum(QuantumRange*g);
  *blue=ClampToQuantum(QuantumRange*b);
}


MagickExport void ConvertRGBToHWB(const Quantum red,const Quantum green,
  const Quantum blue,double *hue,double *whiteness,double *blackness)
{
  double
    b,
    f,
    g,
    p,
    r,
    v,
    w;

  /*
    Convert RGB to HWB colorspace.
  */
  assert(hue != (double *) NULL);
  assert(whiteness != (double *) NULL);
  assert(blackness != (double *) NULL);
  r=(double) red;
  g=(double) green;
  b=(double) blue;
  w=MagickMin(r,MagickMin(g,b));
  v=MagickMax(r,MagickMax(g,b));
  *blackness=1.0-QuantumScale*v;
  *whiteness=QuantumScale*w;
  if (fabs(v-w) < MagickEpsilon)
    {
      *hue=(-1.0);
      return;
    }
  f=(fabs(r-w) < MagickEpsilon) ? g-b : ((fabs(g-w) < MagickEpsilon) ? b-r : r-g);
  p=(fabs(r-w) < MagickEpsilon) ? 3.0 : ((fabs(g-w) < MagickEpsilon) ? 5.0 : 1.0);
  *hue=(p-f/(v-1.0*w))/6.0;
}


MagickExport void ConvertLCHabToRGB(const double luma,const double chroma,
  const double hue,Quantum *red,Quantum *green,Quantum *blue)
{
  double
    X,
    Y,
    Z;

  /*
    Convert LCHab to RGB colorspace.
  */
  assert(red != (Quantum *) NULL);
  assert(green != (Quantum *) NULL);
  assert(blue != (Quantum *) NULL);
  ConvertLCHabToXYZ(100.0*luma,255.0*(chroma-0.5),360.0*hue,&X,&Y,&Z);
  ConvertXYZToRGB(X,Y,Z,red,green,blue);
}


srs_parse_guarded(srs_t *srs, char *buf, int buflen, char *senduser)
{
	char	*srshash;
	char	*srshost;
	char	*srsuser;
	int		 ret;

	if (strncasecmp(senduser, SRS1TAG, 4) == 0) {
		srshash = senduser + 5;
		if (!STRINGP(srshash))
			return SRS_ENOSRS1HASH;
		srshost = strchr(srshash, SRSSEP);
		if (!STRINGP(srshost))
			return SRS_ENOSRS1HOST;
		*srshost++ = '\0';
		srsuser = strchr(srshost, SRSSEP);
		if (!STRINGP(srsuser))
			return SRS_ENOSRS1USER;
		*srsuser++ = '\0';
		ret = srs_hash_check(srs, srshash, 2, srshost, srsuser);
		if (ret != SRS_SUCCESS)
			return ret;
		sprintf(buf, SRS0TAG "%s@%s", srsuser, srshost);
		return SRS_SUCCESS;
	}
	else {
		return srs_parse_shortcut(srs, buf, buflen, senduser);
	}
}

SECURITY: Fix potential denial of service attack against PostSRSd

I discovered that PostSRSd could be tricked into consuming a lot of CPU
time with an SRS address that has an excessively long time stamp tag,
e.g.

SRS0=HHHH=TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT=0@example.com

#  define strncasecmp _strnicmp

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_compile_guarded(srs_t *srs,
				char *buf, int buflen,
				char *sendhost, char *senduser,
				const char *aliashost) {
	char	*srshost;
	char	*srsuser;
	char	*srshash;
	int		 len;
	int		 ret;

	if ((strncasecmp(senduser, SRS1TAG, 4) == 0) &&
		(strchr(srs_separators, senduser[4]) != NULL)) {
		/* Used as a temporary convenience var */
		srshash = senduser + 5;
		if (*srshash == '\0')
			return SRS_ENOSRS1HASH;
		/* Used as a temporary convenience var */
		srshost = strchr(srshash, SRSSEP);
		if (!STRINGP(srshost))
			return SRS_ENOSRS1HOST;
		*srshost++ = '\0';
		srsuser = strchr(srshost, SRSSEP);
		if (!STRINGP(srsuser))
			return SRS_ENOSRS1USER;
		*srsuser++ = '\0';
		srshash = alloca(srs->hashlength + 1);
		ret = srs_hash_create(srs, srshash, 2, srshost, srsuser);
		if (ret != SRS_SUCCESS)
			return ret;
		len = strlen(SRS1TAG) + 1 +
			srs->hashlength + 1 +
				strlen(srshost) + 1 + strlen(srsuser)
			+ 1 + strlen(aliashost);
		if (len >= buflen)
			return SRS_EBUFTOOSMALL;
		sprintf(buf, SRS1TAG "%c%s%c%s%c%s@%s", srs->separator,
						srshash, SRSSEP,
							srshost, SRSSEP, srsuser,
								aliashost);
		return SRS_SUCCESS;
	}
	else if ((strncasecmp(senduser, SRS0TAG, 4) == 0) &&
		(strchr(srs_separators, senduser[4]) != NULL)) {
		srsuser = senduser + 4;
		srshost = sendhost;
		srshash = alloca(srs->hashlength + 1);
		ret = srs_hash_create(srs, srshash, 2, srshost, srsuser);
		if (ret != SRS_SUCCESS)
			return ret;
		len = strlen(SRS1TAG) + 1 +
			srs->hashlength + 1 +
				strlen(srshost) + 1 + strlen(srsuser)
			+ 1 + strlen(aliashost);
		if (len >= buflen)
			return SRS_EBUFTOOSMALL;
		sprintf(buf, SRS1TAG "%c%s%c%s%c%s@%s", srs->separator,
						srshash, SRSSEP,
							srshost, SRSSEP, srsuser,
								aliashost);
	}
	else {
		return srs_compile_shortcut(srs, buf, buflen,
						sendhost, senduser, aliashost);
	}

	return SRS_SUCCESS;
}

#  define strncasecmp _strnicmp

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_hash_create(srs_t *srs, char *buf, int nargs, ...)
{
	va_list	 ap;

	if (srs->numsecrets == 0)
		return SRS_ENOSECRETS;
	if (srs->secrets == NULL)
		return SRS_ENOSECRETS;
	if (srs->secrets[0] == NULL)
		return SRS_ENOSECRETS;

	va_start(ap, nargs);
	srs_hash_create_v(srs, 0, buf, nargs, ap);
	va_end(ap);

	return SRS_SUCCESS;
}

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_timestamp_create(srs_t *srs __attribute__((unused)), char *buf, time_t now)
{
	now = now / SRS_TIME_PRECISION;
	buf[1] = SRS_TIME_BASECHARS[now & ((1 << SRS_TIME_BASEBITS) - 1)];
	now = now >> SRS_TIME_BASEBITS;
	buf[0] = SRS_TIME_BASECHARS[now & ((1 << SRS_TIME_BASEBITS) - 1)];
	buf[2] = '\0';
	return SRS_SUCCESS;
}

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_PRECISION	(60 * 60 * 24)	/* One day */

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

#define SRS_TIME_SLOTS		(1<<(SRS_TIME_BASEBITS<<(SRS_TIME_SIZE-1)))

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_forward_alloc(srs_t *srs, char **sptr,
				const char *sender, const char *alias)
{
	char	*buf;
	int		 slen;
	int		 alen;
	int		 len;
	int		 ret;

	if (srs->noforward)
		return SRS_ENOTREWRITTEN;

	slen = strlen(sender);
	alen = strlen(alias);

	/* strlen(SRSxTAG) + strlen("====+@") < 64 */
	len = slen + alen + srs->hashlength + SRS_TIME_SIZE + 64;
	buf = (char *)srs_f_malloc(len);

	ret = srs_forward(srs, buf, len, sender, alias);

	if (ret == SRS_SUCCESS)
		*sptr = buf;
	else
		srs_f_free(buf);

	return ret;
}

static const char *srs_separators = "=-+";

static srs_malloc_t		srs_f_malloc	= malloc;

static srs_free_t		srs_f_free		= free;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

#define SRS_TIME_SIZE		2

#define SRS_TIME_SLOTS		(1<<(SRS_TIME_BASEBITS<<(SRS_TIME_SIZE-1)))

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_hash_create_v(srs_t *srs, int idx, char *buf, int nargs, va_list ap)
{
#ifdef USE_OPENSSL
	HMAC_CTX		 ctx;
	int				 srshashlen;
	char			 srshash[EVP_MAX_MD_SIZE + 1];
#else
	srs_hmac_ctx_t	 ctx;
	char			 srshash[SHA_DIGESTSIZE + 1];
#endif
	char			*secret;
	char			*data;
	int				 len;
	char			*lcdata;
	unsigned char	*hp;
	char			*bp;
	int				 i;
	int				 j;

	secret = srs->secrets[idx];

#ifdef USE_OPENSSL
	HMAC_CTX_init(&ctx);
	HMAC_Init(&ctx, secret, strlen(secret), EVP_sha1());
#else
	srs_hmac_init(&ctx, secret, strlen(secret));
#endif

	for (i = 0; i < nargs; i++) {
		data = va_arg(ap, char *);
		len = strlen(data);
		lcdata = alloca(len + 1);
		for (j = 0; j < len; j++) {
			if (isupper(data[j]))
				lcdata[j] = tolower(data[j]);
			else
				lcdata[j] = data[j];
		}
#ifdef USE_OPENSSL
		HMAC_Update(&ctx, lcdata, len);
#else
		srs_hmac_update(&ctx, lcdata, len);
#endif
	}

#ifdef USE_OPENSSL
	HMAC_Final(&ctx, srshash, &srshashlen);
	HMAC_CTX_cleanup(&ctx);
	srshash[EVP_MAX_MD_SIZE] = '\0';
#else
	srs_hmac_fini(&ctx, srshash);
	srshash[SHA_DIGESTSIZE] = '\0';
#endif

	/* A little base64 encoding. Just a little. */
	hp = (unsigned char *)srshash;
	bp = buf;
	for (i = 0; i < srs->hashlength; i++) {
		switch (i & 0x03) {
			default:	/* NOTREACHED */
			case 0:
				j = (*hp >> 2);
				break;
			case 1:
				j = ((*hp & 0x03) << 4) |
						((*(hp + 1) & 0xF0) >> 4);
				hp++;
				break;
			case 2:
				j = ((*hp & 0x0F) << 2) |
						((*(hp + 1) & 0xC0) >> 6);
				hp++;
				break;
			case 3:
				j = (*hp++ & 0x3F);
				break;
		}
		*bp++ = SRS_HASH_BASECHARS[j];
	}

	*bp = '\0';
	buf[srs->hashlength] = '\0';
}

#define EVP_MAX_MD_SIZE (16+20) /* The SSLv3 md5+sha1 type */

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_init(srs_t *srs)
{
	memset(srs, 0, sizeof(srs_t));
	srs->secrets = NULL;
	srs->numsecrets = 0;
	srs->separator = '=';
	srs->maxage = 21;
	srs->hashlength = 4;
	srs->hashmin = srs->hashlength;
	srs->alwaysrewrite = FALSE;
}

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}


srs_get_separator(srs_t *srs)
{
	return srs->separator;
}

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}


srs_reverse(srs_t *srs, char *buf, unsigned buflen, const char *sender)
{
	char	*senduser;
	char	*tmp;
	unsigned		 len;

	if (!SRS_IS_SRS_ADDRESS(sender))
		return SRS_ENOTSRSADDRESS;

	if (srs->noreverse)
		return SRS_ENOTREWRITTEN;

	len = strlen(sender);
	if (len >= buflen)
		return SRS_EBUFTOOSMALL;
	senduser = alloca(len + 1);
	strcpy(senduser, sender);

	/* We don't really care about the host for reversal. */
	tmp = strchr(senduser, '@');
	if (tmp != NULL)
		*tmp = '\0';
	return srs_parse_guarded(srs, buf, buflen, senduser);
}

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_hash_check(srs_t *srs, char *hash, int nargs, ...)
{
	va_list	ap;
	char	*srshash;
	char	*tmp;
	int		 len;
	int		 i;

	len = strlen(hash);
	if (len < srs->hashmin)
		return SRS_EHASHTOOSHORT;
	if (len > srs->hashlength) {
		tmp = alloca(srs->hashlength + 1);
		strncpy(tmp, hash, srs->hashlength);
		tmp[srs->hashlength] = '\0';
		hash = tmp;
		len = srs->hashlength;
	}

	for (i = 0; i < srs->numsecrets; i++) {
		va_start(ap, nargs);
		srshash = alloca(srs->hashlength + 1);
		srs_hash_create_v(srs, i, srshash, nargs, ap);
		va_end(ap);
		if (strncasecmp(hash, srshash, len) == 0)
			return SRS_SUCCESS;
	}

	return SRS_EHASHINVALID;
}

#  define strncasecmp _strnicmp

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_set_malloc(srs_malloc_t m, srs_realloc_t r, srs_free_t f)
{
	srs_f_malloc = m;
	srs_f_realloc = r;
	srs_f_free = f;
	return SRS_SUCCESS;
}

static srs_malloc_t		srs_f_malloc	= malloc;

static srs_realloc_t	srs_f_realloc	= realloc;

static srs_free_t		srs_f_free		= free;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}


srs_get_secret(srs_t *srs, int idx)
{
	if (idx < srs->numsecrets)
		return srs->secrets[idx];
	return NULL;
}


srs_add_secret(srs_t *srs, const char *secret)
{
	int		newlen = (srs->numsecrets + 1) * sizeof(char *);
	srs->secrets = (char **)srs_f_realloc(srs->secrets, newlen);
	srs->secrets[srs->numsecrets++] = strdup(secret);
	return SRS_SUCCESS;
}

static const char *srs_separators = "=-+";

static srs_realloc_t	srs_f_realloc	= realloc;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_timestamp_check(srs_t *srs, const char *stamp)
{
	const char	*sp;
	char		*bp;
	int			 off;
	time_t		 now;
	time_t		 then;

	if (strlen(stamp) != 2) return SRS_ETIMESTAMPOUTOFDATE;
	/* We had better go around this loop exactly twice! */
	then = 0;
	for (sp = stamp; *sp; sp++) {
		bp = strchr(SRS_TIME_BASECHARS, toupper(*sp));
		if (bp == NULL)
			return SRS_EBADTIMESTAMPCHAR;
		off = bp - SRS_TIME_BASECHARS;
		then = (then << SRS_TIME_BASEBITS) | off;
	}

	time(&now);
	now = (now / SRS_TIME_PRECISION) % SRS_TIME_SLOTS;
	while (now < then)
		now = now + SRS_TIME_SLOTS;

	if (now <= then + srs->maxage)
		return SRS_SUCCESS;
	return SRS_ETIMESTAMPOUTOFDATE;
}

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_PRECISION	(60 * 60 * 24)	/* One day */

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

#define SRS_TIME_SLOTS		(1<<(SRS_TIME_BASEBITS<<(SRS_TIME_SIZE-1)))

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_forward(srs_t *srs, char *buf, unsigned buflen,
				const char *sender, const char *alias)
{
	char	*senduser;
	char	*sendhost;
	char	*tmp;
	unsigned		 len;

	if (srs->noforward)
		return SRS_ENOTREWRITTEN;

	/* This is allowed to be a plain domain */
	while ((tmp = strchr(alias, '@')) != NULL)
		alias = tmp + 1;

	tmp = strchr(sender, '@');
	if (tmp == NULL)
		return SRS_ENOSENDERATSIGN;
	sendhost = tmp + 1;

	len = strlen(sender);

	if (! srs->alwaysrewrite) {
		if (strcasecmp(sendhost, alias) == 0) {
			if (strlen(sender) >= buflen)
				return SRS_EBUFTOOSMALL;
			strcpy(buf, sender);
			return SRS_SUCCESS;
		}
	}

	/* Reconstruct the whole show into our alloca() buffer. */
	senduser = alloca(len + 1);
	strcpy(senduser, sender);
	tmp = (senduser + (tmp - sender));
	sendhost = tmp + 1;
	*tmp = '\0';

	return srs_compile_guarded(srs, buf, buflen,
					sendhost, senduser, alias);
}

#  define strcasecmp _stricmp

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_set_separator(srs_t *srs, char value)
{
	if (strchr(srs_separators, value) == NULL)
		return SRS_ESEPARATORINVALID;
	srs->separator = value;
	return SRS_SUCCESS;
}


srs_parse_shortcut(srs_t *srs, char *buf, unsigned buflen, char *senduser)
{
	char	*srshash;
	char	*srsstamp;
	char	*srshost;
	char	*srsuser;
	int		 ret;

	if (strncasecmp(senduser, SRS0TAG, 4) == 0) {
		srshash = senduser + 5;
		if (!STRINGP(srshash))
			return SRS_ENOSRS0HASH;
		srsstamp = strchr(srshash, SRSSEP);
		if (!STRINGP(srsstamp))
			return SRS_ENOSRS0STAMP;
		*srsstamp++ = '\0';
		srshost = strchr(srsstamp, SRSSEP);
		if (!STRINGP(srshost))
			return SRS_ENOSRS0HOST;
		*srshost++ = '\0';
		srsuser = strchr(srshost, SRSSEP);
		if (!STRINGP(srsuser))
			return SRS_ENOSRS0USER;
		*srsuser++ = '\0';
		ret = srs_timestamp_check(srs, srsstamp);
		if (ret != SRS_SUCCESS)
			return ret;
		ret = srs_hash_check(srs, srshash, 3, srsstamp,
						srshost, srsuser);
		if (ret != SRS_SUCCESS)
			return ret;
		snprintf(buf, buflen, "%s@%s", srsuser, srshost);
		return SRS_SUCCESS;
	}

	return SRS_ENOTSRSADDRESS;
}


srs_new()
{
	srs_t	*srs = (srs_t *)srs_f_malloc(sizeof(srs_t));
	srs_init(srs);
	return srs;
}

static srs_malloc_t		srs_f_malloc	= malloc;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}


srs_reverse_alloc(srs_t *srs, char **sptr, const char *sender)
{
	char	*buf;
	int		 len;
	int		 ret;

	*sptr = NULL;

	if (!SRS_IS_SRS_ADDRESS(sender))
		return SRS_ENOTSRSADDRESS;

	if (srs->noreverse)
		return SRS_ENOTREWRITTEN;

	len = strlen(sender) + 1;
	buf = (char *)srs_f_malloc(len);

	ret = srs_reverse(srs, buf, len, sender);

	if (ret == SRS_SUCCESS)
		*sptr = buf;
	else
		srs_f_free(buf);

	return ret;
}

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

static srs_malloc_t		srs_f_malloc	= malloc;

static srs_free_t		srs_f_free		= free;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


srs_strerror(int code)
{
	switch (code) {
		/* Simple errors */
		case SRS_SUCCESS:
			return "Success";
		case SRS_ENOTSRSADDRESS:
			return "Not an SRS address.";

		/* Config errors */
		case SRS_ENOSECRETS:
			return "No secrets in SRS configuration.";
		case SRS_ESEPARATORINVALID:
			return "Invalid separator suggested.";

		/* Input errors */
		case SRS_ENOSENDERATSIGN:
			return "No at sign in sender address";
		case SRS_EBUFTOOSMALL:
			return "Buffer too small.";

		/* Syntax errors */
		case SRS_ENOSRS0HOST:
			return "No host in SRS0 address.";
		case SRS_ENOSRS0USER:
			return "No user in SRS0 address.";
		case SRS_ENOSRS0HASH:
			return "No hash in SRS0 address.";
		case SRS_ENOSRS0STAMP:
			return "No timestamp in SRS0 address.";
		case SRS_ENOSRS1HOST:
			return "No host in SRS1 address.";
		case SRS_ENOSRS1USER:
			return "No user in SRS1 address.";
		case SRS_ENOSRS1HASH:
			return "No hash in SRS1 address.";
		case SRS_EBADTIMESTAMPCHAR:
			return "Bad base32 character in timestamp.";
		case SRS_EHASHTOOSHORT:
			return "Hash too short in SRS address.";

		/* SRS errors */
		case SRS_ETIMESTAMPOUTOFDATE:
			return "Time stamp out of date.";
		case SRS_EHASHINVALID:
			return "Hash invalid in SRS address.";

		default:
			return "Unknown error in SRS library.";
	}
}


srs_free(srs_t *srs)
{
	int	 i;
	for (i = 0; i < srs->numsecrets; i++) {
		memset(srs->secrets[i], 0, strlen(srs->secrets[i]));
		srs_f_free(srs->secrets[i]);
		srs->secrets[i] = 0;
	}
	srs_f_free(srs);
}

static srs_free_t		srs_f_free		= free;

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */


srs_compile_shortcut(srs_t *srs,
				char *buf, int buflen,
				char *sendhost, char *senduser,
				const char *aliashost) {
	char	*srshash;
	char	 srsstamp[SRS_TIME_SIZE + 1];
	int		 len;
	int		 ret;

	/* This never happens if we get called from guarded() */
	if ((strncasecmp(senduser, SRS0TAG, 4) == 0) &&
		(strchr(srs_separators, senduser[4]) != NULL)) {
		sendhost = senduser + 5;
		if (*sendhost == '\0')
			return SRS_ENOSRS0HOST;
		senduser = strchr(sendhost, SRSSEP);
		if ((senduser == NULL) || (*senduser == '\0'))
			return SRS_ENOSRS0USER;
	}

	len = strlen(SRS0TAG) + 1 +
		srs->hashlength + 1 +
			SRS_TIME_SIZE + 1 +
				strlen(sendhost) + 1 + strlen(senduser)
			+ 1 + strlen(aliashost);
	if (len >= buflen)
		return SRS_EBUFTOOSMALL;

	ret = srs_timestamp_create(srs, srsstamp, time(NULL));
	if (ret != SRS_SUCCESS)
		return ret;
	srshash = alloca(srs->hashlength + 1);
	ret = srs_hash_create(srs, srshash,3, srsstamp, sendhost, senduser);
	if (ret != SRS_SUCCESS)
		return ret;

	sprintf(buf, SRS0TAG "%c%s%c%s%c%s%c%s@%s", srs->separator,
					srshash, SRSSEP, srsstamp, SRSSEP,
						sendhost, SRSSEP, senduser,
							aliashost);

	return SRS_SUCCESS;
}

#  define strncasecmp _strnicmp

#define STRINGP(s) ((s != NULL) && (*(s) != '\0'))

static const char *srs_separators = "=-+";

#define SRS_PARAM_DEFINE(n, t) \
	int srs_set_ ## n (srs_t *srs, t value) { \
		srs->n = value; \
		return SRS_SUCCESS; \
	} \
	t srs_get_ ## n (srs_t *srs) { \
		return srs->n; \
	}

#define SRS_TIME_BASEBITS	5		/* 2^5 = 32 = strlen(CHARS) */

const char *SRS_TIME_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";

#define SRS_TIME_SIZE		2

#define SRS_TIME_SLOTS		(1<<(SRS_TIME_BASEBITS<<(SRS_TIME_SIZE-1)))

const char *SRS_HASH_BASECHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
								 "abcdefghijklmnopqrstuvwxyz"
								 "0123456789+/";


InitDA(qqueue_t *pThis, int bLockMutex)
{
	DEFiRet;
	DEFVARS_mutexProtection;
	uchar pszBuf[64];
	size_t lenBuf;

	BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex);
	/* check if we already have a DA worker pool. If not, initiate one. Please note that the
	 * pool is created on first need but never again destructed (until the queue is). This
	 * is intentional. We assume that when we need it once, we may also need it on another
	 * occasion. Ressources used are quite minimal when no worker is running.
	 * rgerhards, 2008-01-24
	 * NOTE: this is the DA worker *pool*, not the DA queue!
	 */
	lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis));
	CHKiRet(wtpConstruct		(&pThis->pWtpDA));
	CHKiRet(wtpSetDbgHdr		(pThis->pWtpDA, pszBuf, lenBuf));
	CHKiRet(wtpSetpfChkStopWrkr	(pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA));
	CHKiRet(wtpSetpfGetDeqBatchSize	(pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize));
	CHKiRet(wtpSetpfDoWork		(pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA));
	CHKiRet(wtpSetpfObjProcessed	(pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed));
	CHKiRet(wtpSetpmutUsr		(pThis->pWtpDA, pThis->mut));
	CHKiRet(wtpSetpcondBusy		(pThis->pWtpDA, &pThis->notEmpty));
	CHKiRet(wtpSetiNumWorkerThreads	(pThis->pWtpDA, 1));
	CHKiRet(wtpSettoWrkShutdown	(pThis->pWtpDA, pThis->toWrkShutdown));
	CHKiRet(wtpSetpUsr		(pThis->pWtpDA, pThis));
	CHKiRet(wtpConstructFinalize	(pThis->pWtpDA));
	/* if we reach this point, we have a "good" DA worker pool */

	/* now construct the actual queue (if it does not already exist) */
	if(pThis->pqDA == NULL) {
		CHKiRet(StartDA(pThis));
	}

finalize_it:
	END_MTX_PROTECTED_OPERATIONS(pThis->mut);
	RETiRet;
}

bugfix: memory leak when $RepeatedMsgReduction on was used

bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=225

static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates);

static rsRetVal RateLimiter(qqueue_t *pThis);

static int qqueueChkStopWrkrDA(qqueue_t *pThis);

static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal);

static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti);

static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti);

static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);


getPhysicalQueueSize(qqueue_t *pThis)
{
	return pThis->iQueueSize;
}


static inline rsRetVal tdlPop(qqueue_t *pQueue)
{
	toDeleteLst_t *pRemove;
	DEFiRet;

	ISOBJ_TYPE_assert(pQueue, qqueue);
	assert(pQueue->toDeleteLst != NULL);

	pRemove = pQueue->toDeleteLst;
	pQueue->toDeleteLst = pQueue->toDeleteLst->pNext;
	free(pRemove);

	RETiRet;
}


static rsRetVal qConstructDisk(qqueue_t *pThis)
{
	DEFiRet;
	int bRestarted = 0;

	ASSERT(pThis != NULL);

	/* and now check if there is some persistent information that needs to be read in */
	iRet = qqueueTryLoadPersistedInfo(pThis);
	if(iRet == RS_RET_OK)
		bRestarted = 1;
	else if(iRet != RS_RET_FILE_NOT_FOUND)
			FINALIZE;

	if(bRestarted == 1) {
		;
	} else {
		CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite));
		CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles));
		CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
		CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000));
		CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE));
		CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR));
		CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite));

		CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq));
		CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0));
		CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
		CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000));
		CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ));
		CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR));
		CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq));

		CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel));
		CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles));
		CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1));
		CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
		CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000));
		CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ));
		CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR));
		CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDel));

		CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite,   pThis->pszFilePrefix, pThis->lenFilePrefix));
		CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix));
		CHKiRet(strm.SetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix));
	}

	/* now we set (and overwrite in case of a persisted restart) some parameters which
	 * should always reflect the current configuration variables. Be careful by doing so,
	 * for example file name generation must not be changed as that would break the
	 * ability to read existing queue files. -- rgerhards, 2008-01-12
	 */
	CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize));
	CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize));
	CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize));

finalize_it:
	RETiRet;
}


static rsRetVal qDelFixedArray(qqueue_t *pThis)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	pThis->tVars.farray.head++;
	if (pThis->tVars.farray.head == pThis->iMaxQueueSize)
		pThis->tVars.farray.head = 0;

	RETiRet;
}


	if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) {
		CHKiRet(DoSaveOnShutdown(pThis));
	}


qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix)
{
	DEFiRet;

	free(pThis->pszFilePrefix);
	pThis->pszFilePrefix = NULL;

	if(pszPrefix == NULL) /* just unset the prefix! */
		ABORT_FINALIZE(RS_RET_OK);

	if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL)
		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
	memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1);
	pThis->lenFilePrefix = iLenPrefix;

finalize_it:
	RETiRet;
}


static rsRetVal qConstructLinkedList(qqueue_t *pThis)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	pThis->tVars.linklist.pDeqRoot = NULL;
	pThis->tVars.linklist.pDelRoot = NULL;
	pThis->tVars.linklist.pLast = NULL;

	qqueueChkIsDA(pThis);

	RETiRet;
}


qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);

	if(iMaxFileSize < 1024) {
		ABORT_FINALIZE(RS_RET_VALUE_TOO_LOW);
	}

	pThis->iMaxFileSize = iMaxFileSize;

finalize_it:
	RETiRet;
}


static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis)
{
	return RS_RET_OK;
}


qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub)
{
	int iCancelStateSave;
	int i;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	assert(pMultiSub != NULL);

	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
	d_pthread_mutex_lock(pThis->mut);
	for(i = 0 ; i < pMultiSub->nElem ; ++i) {
		CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i]));
	}
	qqueueChkPersist(pThis, pMultiSub->nElem);

finalize_it:
	/* make sure at least one worker is running. */
	qqueueAdviseMaxWorkers(pThis);
	/* and release the mutex */
	d_pthread_mutex_unlock(pThis->mut);
	pthread_setcancelstate(iCancelStateSave, NULL);
	DBGOPRINT((obj_t*) pThis, "MultiEnqObj advised worker start\n");

	RETiRet;
}


static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in)
{
	DEFiRet;

	ASSERT(pThis != NULL);
	pThis->tVars.farray.pBuf[pThis->tVars.farray.tail] = in;
	pThis->tVars.farray.tail++;
	if (pThis->tVars.farray.tail == pThis->iMaxQueueSize)
		pThis->tVars.farray.tail = 0;

	RETiRet;
}


qqueueEnqObjDirect(qqueue_t *pThis, void *pUsr)
{
	DEFiRet;
	ISOBJ_TYPE_assert(pThis, qqueue);
	iRet = qAddDirect(pThis, pUsr);
	RETiRet;
}


ShutdownWorkers(qqueue_t *pThis)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */

	DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n");

	CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis));

	if(getPhysicalQueueSize(pThis) > 0) {
		CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis));
	}

	CHKiRet(cancelWorkers(pThis));

	/* ... finally ... all worker threads have terminated :-)
	 * Well, more precisely, they *are in termination*. Some cancel cleanup handlers
	 * may still be running. Note that the main queue's DA worker may still be running.
	 */
	DBGOPRINT((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n",
		  getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));

finalize_it:
	RETiRet;
}


	CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) {
		DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet);
	}

static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates);

static rsRetVal RateLimiter(qqueue_t *pThis);

static int qqueueChkStopWrkrDA(qqueue_t *pThis);

static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal);

static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti);

static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti);

static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

#define QUEUE_NO_CHECKPOINT	0


ConsumerReg(qqueue_t *pThis, wti_t *pWti)
{
	int iCancelStateSave;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ISOBJ_TYPE_assert(pWti, wti);

	CHKiRet(DequeueForConsumer(pThis, pWti));

	/* we now have a non-idle batch of work, so we can release the queue mutex and process it */
	d_pthread_mutex_unlock(pThis->mut);

	/* at this spot, we may be cancelled */
	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);

	CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate));

	/* we now need to check if we should deliberately delay processing a bit
	 * and, if so, do that. -- rgerhards, 2008-01-30
	 */
//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that...
	if(pThis->iDeqSlowdown) {
		DBGOPRINT((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n",
			  pThis->iDeqSlowdown);
		srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000);
	}

	/* but now cancellation is no longer permitted */
	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);

	/* now we are done, but need to re-aquire the mutex */
	d_pthread_mutex_lock(pThis->mut);

finalize_it:
	dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet,
	          getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
	RETiRet;
}


	if(pThis->pWtpDA != NULL) {
		wtpDestruct(&pThis->pWtpDA);
	}


static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates)
{
	DEFiRet;
	ISOBJ_TYPE_assert(pThis, qqueue);
	assert(nUpdates >= 0);

	if(nUpdates == 0)
		FINALIZE;

	pThis->iUpdsSincePersist += nUpdates;
	if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) {
		qqueuePersist(pThis, QUEUE_CHECKPOINT);
		pThis->iUpdsSincePersist = 0;
	}

finalize_it:
	RETiRet;
}

static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates);

static rsRetVal RateLimiter(qqueue_t *pThis);

static int qqueueChkStopWrkrDA(qqueue_t *pThis);

static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal);

static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti);

static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti);

static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

#define QUEUE_CHECKPOINT	1


ChkStopWrkrReg(qqueue_t *pThis)
{
	DEFiRet;
	if(pThis->bEnqOnly) {
		iRet = RS_RET_TERMINATE_NOW;
	} else if(pThis->pqParent != NULL) {
		iRet = RS_RET_TERMINATE_WHEN_IDLE;
	}

	RETiRet;
}


	if(pThis->pqParent == NULL) {
		/* if we are not a child, we allocated our own mutex, which we now need to destroy */
		pthread_mutex_destroy(pThis->mut);
		free(pThis->mut);
	}


cancelWorkers(qqueue_t *pThis)
{
	rsRetVal iRetLocal;
	DEFiRet;

	/* Now queue workers should have terminated. If not, we need to cancel them as we have applied
	 * all timeout setting. If any worker in any queue still executes, its consumer is possibly
	 * long-running and cancelling is the only way to get rid of it.
	 */
	DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n");
	iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */
	if(iRetLocal != RS_RET_OK) {
		DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel primary queue worker "
			  "threads, continuing, but results are unpredictable\n", iRetLocal);
	}

	/* ... and now the DA queue, if it exists (should always be after the primary one) */
	if(pThis->pqDA != NULL) {
		DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n");
		iRetLocal = wtpCancelAll(pThis->pqDA->pWtpReg); /* returns immediately if all threads already have terminated */
		if(iRetLocal != RS_RET_OK) {
			DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker "
				  "threads, continuing, but results are unpredictable\n", iRetLocal);
		}

		/* finally, we cancel the main queue's DA worker pool, if it still is running. It may be
		 * restarted later to persist the queue. But we stop it, because otherwise we get into
		 * big trouble when resetting the logical dequeue pointer. This operation can only be
		 * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28
		 */
		DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n");
		wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */
	}

	RETiRet;
}


static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq)
{
	toDeleteLst_t *pNew;
	toDeleteLst_t *pPrev;
	DEFiRet;

	ISOBJ_TYPE_assert(pQueue, qqueue);
	assert(pQueue->toDeleteLst != NULL);

	CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t)));
	pNew->deqID = deqID;
	pNew->nElemDeq = nElemDeq;

	/* now find right spot */
	for(  pPrev = pQueue->toDeleteLst
	    ; pPrev != NULL && deqID > pPrev->deqID
	    ; pPrev = pPrev->pNext) {
		/*JUST SEARCH*/;
	}

	if(pPrev == NULL) {
		pNew->pNext = pQueue->toDeleteLst;
		pQueue->toDeleteLst = pNew;
	} else {
		pNew->pNext = pPrev->pNext;
		pPrev->pNext = pNew;
	}

finalize_it:
	RETiRet;
}


static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr)
{
	DEFiRet;
	number_t nWriteCount;

	ASSERT(pThis != NULL);

	CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount));
	CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite));
	CHKiRet(strm.Flush(pThis->tVars.disk.pWrite));
	CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */

	pThis->tVars.disk.sizeOnDisk += nWriteCount;

	/* we have enqueued the user element to disk. So we now need to destruct
	 * the in-memory representation. The instance will be re-created upon
	 * dequeue. -- rgerhards, 2008-07-09
	 */
	objDestruct(pUsr);

	DBGOPRINT((obj_t*) pThis, "write wrote %lld octets to disk, queue disk size now %lld octets, EnqOnly:%d\n",
		   nWriteCount, pThis->tVars.disk.sizeOnDisk, pThis->bEnqOnly);

finalize_it:
	RETiRet;
}


static rsRetVal qConstructFixedArray(qqueue_t *pThis)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	if(pThis->iMaxQueueSize == 0)
		ABORT_FINALIZE(RS_RET_QSIZE_ZERO);

	if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) {
		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);
	}

	pThis->tVars.farray.deqhead = 0;
	pThis->tVars.farray.head = 0;
	pThis->tVars.farray.tail = 0;

	qqueueChkIsDA(pThis);

finalize_it:
	RETiRet;
}


rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads,
		        int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*))
{
	DEFiRet;
	qqueue_t *pThis;

	ASSERT(ppThis != NULL);
	ASSERT(pConsumer != NULL);
	ASSERT(iWorkerThreads >= 0);

	CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t)));

	/* we have an object, so let's fill the properties */
	objConstructSetObjInfo(pThis);
	if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL)
		ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY);

	/* set some water marks so that we have useful defaults if none are set specifically */
	pThis->iFullDlyMrk  = iMaxQueueSize - (iMaxQueueSize / 100) *  3; /* default 97% */
	pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */

	pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir);
	pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */
	pThis->iQueueSize = 0;
	pThis->nLogDeq = 0;
	pThis->iMaxQueueSize = iMaxQueueSize;
	pThis->pConsumer = pConsumer;
	pThis->iNumWorkerThreads = iWorkerThreads;
	pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */
	pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */

	pThis->pszFilePrefix = NULL;
	pThis->qType = qType;

	/* set type-specific handlers and other very type-specific things (we can not totally hide it...) */
	switch(qType) {
		case QUEUETYPE_FIXED_ARRAY:
			pThis->qConstruct = qConstructFixedArray;
			pThis->qDestruct = qDestructFixedArray;
			pThis->qAdd = qAddFixedArray;
			pThis->qDeq = qDeqFixedArray;
			pThis->qDel = qDelFixedArray;
			pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
			break;
		case QUEUETYPE_LINKEDLIST:
			pThis->qConstruct = qConstructLinkedList;
			pThis->qDestruct = qDestructLinkedList;
			pThis->qAdd = qAddLinkedList;
			pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList;
			pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList;
			pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
			break;
		case QUEUETYPE_DISK:
			pThis->qConstruct = qConstructDisk;
			pThis->qDestruct = qDestructDisk;
			pThis->qAdd = qAddDisk;
			pThis->qDeq = qDeqDisk;
			pThis->qDel = qDelDisk;
			pThis->MultiEnq = qqueueMultiEnqObjNonDirect;
			/* special handling */
			pThis->iNumWorkerThreads = 1; /* we need exactly one worker */
			break;
		case QUEUETYPE_DIRECT:
			pThis->qConstruct = qConstructDirect;
			pThis->qDestruct = qDestructDirect;
			pThis->qAdd = qAddDirect;
			pThis->qDel = qDelDirect;
			pThis->MultiEnq = qqueueMultiEnqObjDirect;
			break;
	}

	INIT_ATOMIC_HELPER_MUT(pThis->mutQueueSize);
	INIT_ATOMIC_HELPER_MUT(pThis->mutLogDeq);

finalize_it:
	OBJCONSTRUCT_CHECK_SUCCESS_AND_CLEANUP
	RETiRet;
}


tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis)
{
	struct timespec tTimeout;
	rsRetVal iRetLocal;
	DEFiRet;

RUNLOG_STR("trying to shutdown workers within Action Timeout");
	ISOBJ_TYPE_assert(pThis, qqueue);
	ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */

	/* instruct workers to finish ASAP, even if still work exists */
	DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode\n");
	pThis->bEnqOnly = 1;
	pThis->bShutdownImmediate = 1;
	/* now DA queue */
	if(pThis->bIsDA) {
		pThis->pqDA->bEnqOnly = 1;
		pThis->pqDA->bShutdownImmediate = 1;
	}

// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance...
	/* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */
	timeoutComp(&tTimeout, pThis->toActShutdown);
	DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n");
	iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
	if(iRetLocal == RS_RET_TIMED_OUT) {
		DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and "
			  "triggers cancellation)\n");
	} else if(iRetLocal != RS_RET_OK) {
		DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue "
			  "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
	}

	if(pThis->pqDA != NULL) {
		/* and now the same for the DA queue */
		DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n");
		iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
		if(iRetLocal == RS_RET_TIMED_OUT) {
			DBGOPRINT((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable "
				  "and triggers cancellation)\n");
		} else if(iRetLocal != RS_RET_OK) {
			DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA "
				  "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal);
		}

		/* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout,
		 * which should be sufficient and usually not be required (it is expected to have finished
		 * long before while we were processing the queue timeout in shutdown phase 1).
		 * rgerhards, 2009-10-14
		 */
		timeoutComp(&tTimeout, 100);
		DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n");
		iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout);
		if(iRetLocal == RS_RET_TIMED_OUT) {
			DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool "
					          "(this is not good, but probably OK)\n");
		} else {
			DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down.\n");
		}
	}

	RETiRet;
}


static inline qDeqID getNextDeqID(qqueue_t *pQueue)
{
	ISOBJ_TYPE_assert(pQueue, qqueue);
	return pQueue->deqIDAdd++;
}


static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr)
{
	qLinkedList_t *pEntry;
	DEFiRet;

	CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t))));

	pEntry->pNext = NULL;
	pEntry->pUsr = pUsr;

	if(pThis->tVars.linklist.pDelRoot == NULL) {
		pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry;
	} else {
		pThis->tVars.linklist.pLast->pNext = pEntry;
		pThis->tVars.linklist.pLast = pEntry;
	}

	if(pThis->tVars.linklist.pDeqRoot == NULL) {
		pThis->tVars.linklist.pDeqRoot = pEntry;
	}

finalize_it:
	RETiRet;
}


static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr)
{
	DEFiRet;

	CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL));

finalize_it:
	RETiRet;
}


tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis)
{
	struct timespec tTimeout;
	rsRetVal iRetLocal;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */

	if(pThis->bIsDA) {
		/* We need to lock the mutex, as otherwise we may have a race that prevents
		 * us from awaking the DA worker. */
		d_pthread_mutex_lock(pThis->mut);

		/* tell regular queue DA worker to stop shuffling messages to DA queue... */
		DBGOPRINT((obj_t*) pThis, "setting EnqOnly mode for DA worker\n");
		pThis->pqDA->bEnqOnly = 1;
		wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE);
		wtpAdviseMaxWorkers(pThis->pWtpDA, 1);
		DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n");

		/* also tell the DA queue worker to shut down, so that it already knows... */
		wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN);
		wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */
		DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n");

		d_pthread_mutex_unlock(pThis->mut);
	}

	/* first calculate absolute timeout - we need the absolute value here, because we need to coordinate
	 * shutdown of both the regular and DA queue on *the same* timeout.
	 */
	timeoutComp(&tTimeout, pThis->toQShutdown);
	DBGOPRINT((obj_t*) pThis, "trying shutdown of regular workers\n");
	iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN, &tTimeout);
	if(iRetLocal == RS_RET_TIMED_OUT) {
		DBGOPRINT((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n");
	} else {
		DBGOPRINT((obj_t*) pThis, "regular queue workers shut down.\n");
	}

	/* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */
	if(pThis->pqDA != NULL) {
		DBGOPRINT((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n",
			 qqueueGetID(pThis->pqDA));
		/* we use the same absolute timeout as above, so we do not use more than the configured
		 * timeout interval!
		 */
		DBGOPRINT((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n");
		iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout);
		if(iRetLocal == RS_RET_TIMED_OUT) {
			DBGOPRINT((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n");
		} else {
			DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n");
		}
	}

	RETiRet;
}


DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch)
{
	toDeleteLst_t *pTdl;
	qDeqID	deqIDDel;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	assert(pBatch != NULL);

	pTdl = tdlPeek(pThis); /* get current head element */
	if(pTdl == NULL) { /* to-delete list empty */
		DoDeleteBatchFromQStore(pThis, pBatch->nElem);
	} else if(pBatch->deqID == pThis->deqIDDel) {
		deqIDDel = pThis->deqIDDel;
		pTdl = tdlPeek(pThis);
		while(pTdl != NULL && deqIDDel == pTdl->deqID) {
			DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq);
			tdlPop(pThis);
			++deqIDDel;
			pTdl = tdlPeek(pThis);
		}
		/* old entries deleted, now delete current ones... */
		DoDeleteBatchFromQStore(pThis, pBatch->nElem);
	} else {
		/* can not delete, insert into to-delete list */
		dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID);
		CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem));
	}

finalize_it:
	RETiRet;
}


ConsumerDA(qqueue_t *pThis, wti_t *pWti)
{
	int i;
	int iCancelStateSave;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ISOBJ_TYPE_assert(pWti, wti);

	CHKiRet(DequeueForConsumer(pThis, pWti));

	/* we now have a non-idle batch of work, so we can release the queue mutex and process it */
	d_pthread_mutex_unlock(pThis->mut);

	/* at this spot, we may be cancelled */
	pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave);

	/* iterate over returned results and enqueue them in DA queue */
	for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) {
		/* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs
		 * the message. So far, we simply assume we always have msg_t, what currently is always the case.
		 * rgerhards, 2009-05-28
		 */
		CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY,
			(obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp))));
		pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */
	}

	/* but now cancellation is no longer permitted */
	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);

	/* now we are done, but need to re-aquire the mutex */
	d_pthread_mutex_lock(pThis->mut);

finalize_it:
	DBGOPRINT((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet);
	RETiRet;
}


static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint)
{
	DEFiRet;
	strm_t *psQIF = NULL; /* Queue Info File */
	uchar pszQIFNam[MAXFNAME];
	size_t lenQIFNam;

	ASSERT(pThis != NULL);

	if(pThis->qType != QUEUETYPE_DISK) {
		if(getPhysicalQueueSize(pThis) > 0) {
			/* This error code is OK, but we will probably not implement this any time
 			 * The reason is that persistence happens via DA queues. But I would like to
			 * leave the code as is, as we so have a hook in case we need one.
			 * -- rgerhards, 2008-01-28
			 */
			ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED);
		} else
			FINALIZE; /* if the queue is empty, we are happy and done... */
	}

	DBGOPRINT((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis));

	/* Construct file name */
	lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi",
			     (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix);

	if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) {
		if(pThis->bNeedDelQIF) {
			unlink((char*)pszQIFNam);
			pThis->bNeedDelQIF = 0;
		}
		/* indicate spool file needs to be deleted */
		if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */
			CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1));
		FINALIZE; /* nothing left to do, so be happy */
	}

	CHKiRet(strm.Construct(&psQIF));
	CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC));
	CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles));
	CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE));
	CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam));
	CHKiRet(strm.ConstructFinalize(psQIF));

	/* first, write the property bag for ourselfs
	 * And, surprisingly enough, we currently need to persist only the size of the
	 * queue. All the rest is re-created with then-current config parameters when the
	 * queue is re-created. Well, we'll also save the current queue type, just so that
	 * we know when somebody has changed the queue type... -- rgerhards, 2008-01-11
	 */
	CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis));
	objSerializeSCALAR(psQIF, iQueueSize, INT);
	objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64);
	objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64);
	CHKiRet(obj.EndSerialize(psQIF));

	/* now persist the stream info */
	CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF));
	CHKiRet(strm.Serialize(pThis->tVars.disk.pReadDel, psQIF));

	/* tell the input file object that it must not delete the file on close if the queue
	 * is non-empty - but only if we are not during a simple checkpoint
	 */
	if(bIsCheckpoint != QUEUE_CHECKPOINT) {
		CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0));
	}

	/* we have persisted the queue object. So whenever it comes to an empty queue,
	 * we need to delete the QIF. Thus, we indicte that need.
	 */
	pThis->bNeedDelQIF = 1;

finalize_it:
	if(psQIF != NULL)
		strm.Destruct(&psQIF);

	RETiRet;
}


rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; }

static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates);

static rsRetVal RateLimiter(qqueue_t *pThis);

static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal);

static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti);

static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti);

static rsRetVal qqueueMultiEnqObjNonDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);

static rsRetVal qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub);


getLogicalQueueSize(qqueue_t *pThis)
{
	return pThis->iQueueSize - pThis->nLogDeq;
}


RateLimiter(qqueue_t *pThis)
{
	DEFiRet;
	int iDelay;
	int iHrCurr;
	time_t tCurr;
	struct tm m;

	ISOBJ_TYPE_assert(pThis, qqueue);

	iDelay = 0;
	if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */
		/* time calls are expensive, so only do them when needed */
		datetime.GetTime(&tCurr);
		localtime_r(&tCurr, &m);
		iHrCurr = m.tm_hour;

		if(pThis->iDeqtWinToHr < pThis->iDeqtWinFromHr) {
			if(iHrCurr < pThis->iDeqtWinToHr || iHrCurr > pThis->iDeqtWinFromHr) {
				; /* do not delay */
			} else {
				iDelay = (pThis->iDeqtWinFromHr - iHrCurr) * 3600;
				/* this time, we are already into the next hour, so we need
				 * to subtract our current minute and seconds.
				 */
				iDelay -= m.tm_min * 60;
				iDelay -= m.tm_sec;
			}
		} else {
			if(iHrCurr >= pThis->iDeqtWinFromHr && iHrCurr < pThis->iDeqtWinToHr) {
				; /* do not delay */
			} else {
				if(iHrCurr < pThis->iDeqtWinFromHr) {
					iDelay = (pThis->iDeqtWinFromHr - iHrCurr - 1) * 3600; /* -1 as we are already in the hour */
					iDelay += (60 - m.tm_min) * 60;
					iDelay += 60 - m.tm_sec;
				} else {
					iDelay = (24 - iHrCurr + pThis->iDeqtWinFromHr) * 3600;
					/* this time, we are already into the next hour, so we need
					 * to subtract our current minute and seconds.
					 */
					iDelay -= m.tm_min * 60;
					iDelay -= m.tm_sec;
				}
			}
		}
	}

	if(iDelay > 0) {
		DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay);
		srSleep(iDelay, 0);
	}

	RETiRet;
}


DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem)
{
	int i;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);

	/* now send delete request to storage driver */
	for(i = 0 ; i < nElem ; ++i) {
		pThis->qDel(pThis);
	}

	/* iQueueSize is not decremented by qDel(), so we need to do it ourselves */
	ATOMIC_SUB(&pThis->iQueueSize, nElem, &pThis->mutQueueSize);
	ATOMIC_SUB(&pThis->nLogDeq, nElem, &pThis->mutLogDeq);
dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
	++pThis->deqIDDel; /* one more batch dequeued */

	RETiRet;
}


static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr)
{
	batch_t singleBatch;
	batch_obj_t batchObj;
	int i;
	DEFiRet;

	//TODO: init batchObj (states _OK and new fields -- CHECK)
	ASSERT(pThis != NULL);

	/* calling the consumer is quite different here than it is from a worker thread */
	/* we need to provide the consumer's return value back to the caller because in direct
	 * mode the consumer probably has a lot to convey (which get's lost in the other modes
	 * because they are asynchronous. But direct mode is deliberately synchronous.
	 * rgerhards, 2008-02-12
	 * We use our knowledge about the batch_t structure below, but without that, we
	 * pay a too-large performance toll... -- rgerhards, 2009-04-22
	 */
	memset(&batchObj, 0, sizeof(batch_obj_t));
	memset(&singleBatch, 0, sizeof(batch_t));
	batchObj.state = BATCH_STATE_RDY;
	batchObj.pUsrp = (obj_t*) pUsr;
	batchObj.bFilterOK = 1;
	singleBatch.nElem = 1; /* there always is only one in direct mode */
	singleBatch.pElem = &batchObj;
	iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate);
	/* delete the batch string params: TODO: create its own "class" for this */
	for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) {
		free(batchObj.staticActStrings[i]);
	}
	objDestruct(pUsr);

	RETiRet;
}


static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr)
{
	qLinkedList_t *pEntry;
	DEFiRet;

	pEntry = pThis->tVars.linklist.pDeqRoot;
	ISOBJ_TYPE_assert(pEntry->pUsr, msg);
	*ppUsr = pEntry->pUsr;
	pThis->tVars.linklist.pDeqRoot = pEntry->pNext;

	RETiRet;
}


static rsRetVal qDelDisk(qqueue_t *pThis)
{
	obj_t *pDummyObj;	/* we need to deserialize it... */
	DEFiRet;

	int64 offsIn;
	int64 offsOut;

	CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn));
	CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL));
	objDestruct(pDummyObj);
	CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut));

	/* This time it is a bit tricky: we free disk space only upon file deletion. So we need
	 * to keep track of what we have read until we get an out-offset that is lower than the
	 * in-offset (which indicates file change). Then, we can subtract the whole thing from
	 * the on-disk size. -- rgerhards, 2008-01-30
	 */
	if(offsIn < offsOut) {
		pThis->tVars.disk.bytesRead += offsOut - offsIn;
	} else {
		pThis->tVars.disk.sizeOnDisk -= pThis->tVars.disk.bytesRead;
		pThis->tVars.disk.bytesRead = offsOut;
		DBGOPRINT((obj_t*) pThis, "a file has been deleted, now %lld octets disk space used\n", pThis->tVars.disk.sizeOnDisk);
		/* awake possibly waiting enq process */
		pthread_cond_signal(&pThis->notFull); /* we hold the mutex while we are in here! */
	}

finalize_it:
	RETiRet;
}


static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis)
{
	return RS_RET_OK;
}


static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis)
{
	DEFiRet;

	queueDrain(pThis); /* discard any remaining queue entries */

	/* with the linked list type, there is nothing left to do here. The
	 * reason is that there are no dynamic elements for the list itself.
	 */

	RETiRet;
}


qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pThis)
{
	DEFiRet;
	ISOBJ_TYPE_assert(pStrm, strm);
	ISOBJ_TYPE_assert(pThis, qqueue);
	CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir())));
finalize_it:
	RETiRet;
}


rsRetVal qqueueEnqObjDirectBatch(qqueue_t *pThis, batch_t *pBatch)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	/* calling the consumer is quite different here than it is from a worker thread */
	/* we need to provide the consumer's return value back to the caller because in direct
	 * mode the consumer probably has a lot to convey (which get's lost in the other modes
	 * because they are asynchronous. But direct mode is deliberately synchronous.
	 * rgerhards, 2008-02-12
	 * We use our knowledge about the batch_t structure below, but without that, we
	 * pay a too-large performance toll... -- rgerhards, 2009-04-22
	 */
	iRet = pThis->pConsumer(pThis->pUsr, pBatch, &pThis->bShutdownImmediate);

	RETiRet;
}


qqueueAdviseMaxWorkers(qqueue_t *pThis)
{
	DEFiRet;
	int iMaxWorkers;

	ISOBJ_TYPE_assert(pThis, qqueue);

	if(!pThis->bEnqOnly) {
		if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) {
			wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */
		} else {
			if(getLogicalQueueSize(pThis) == 0) {
				iMaxWorkers = 0;
			} else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) {
				iMaxWorkers = 1;
			} else {
				iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1;
			}
			wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers);
		}
	}

	RETiRet;
}


qqueueDeq(qqueue_t *pThis, void **ppUsr)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	/* we do NOT abort if we encounter an error, because otherwise the queue
	 * will not be decremented, what will most probably result in an endless loop.
	 * If we decrement, however, we may lose a message. But that is better than
	 * losing the whole process because it loops... -- rgerhards, 2008-01-03
	 */
	iRet = pThis->qDeq(pThis, ppUsr);
	ATOMIC_INC(&pThis->nLogDeq, &pThis->mutLogDeq);

//	DBGOPRINT((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n",
//		  getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));

	RETiRet;
}


qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */
{
	DEFiRet;
	uchar pszBuf[64];
	size_t lenBuf;

	ASSERT(pThis != NULL);

	/* we need to do a quick check if our water marks are set plausible. If not,
	 * we correct the most important shortcomings. TODO: do that!!!! -- rgerhards, 2008-03-14
	 */

	/* finalize some initializations that could not yet be done because it is
	 * influenced by properties which might have been set after queueConstruct ()
	 */
	if(pThis->pqParent == NULL) {
		pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t));
		pthread_mutex_init(pThis->mut, NULL);
	} else {
		/* child queue, we need to use parent's mutex */
		DBGOPRINT((obj_t*) pThis, "I am a child\n");
		pThis->mut = pThis->pqParent->mut;
	}

	pthread_mutex_init(&pThis->mutThrdMgmt, NULL);
	pthread_cond_init (&pThis->condDAReady, NULL);
	pthread_cond_init (&pThis->notFull, NULL);
	pthread_cond_init (&pThis->notEmpty, NULL);
	pthread_cond_init (&pThis->belowFullDlyWtrMrk, NULL);
	pthread_cond_init (&pThis->belowLightDlyWtrMrk, NULL);

	/* call type-specific constructor */
	CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */

	DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, "
				  "full delay %d, light delay %d, deq batch size %d starting\n",
		  pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize,
		  getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis),
		  pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk,
		  pThis->iDeqBatchSize);

	if(pThis->qType == QUEUETYPE_DIRECT)
		FINALIZE;	/* with direct queues, we are already finished... */

	/* create worker thread pools for regular and DA operation.
	 */
	lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis));
	CHKiRet(wtpConstruct		(&pThis->pWtpReg));
	CHKiRet(wtpSetDbgHdr		(pThis->pWtpReg, pszBuf, lenBuf));
	CHKiRet(wtpSetpfRateLimiter	(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter));
	CHKiRet(wtpSetpfChkStopWrkr	(pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg));
	CHKiRet(wtpSetpfGetDeqBatchSize	(pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize));
	CHKiRet(wtpSetpfDoWork		(pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg));
	CHKiRet(wtpSetpfObjProcessed	(pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed));
	CHKiRet(wtpSetpmutUsr		(pThis->pWtpReg, pThis->mut));
	CHKiRet(wtpSetpcondBusy		(pThis->pWtpReg, &pThis->notEmpty));
	CHKiRet(wtpSetiNumWorkerThreads	(pThis->pWtpReg, pThis->iNumWorkerThreads));
	CHKiRet(wtpSettoWrkShutdown	(pThis->pWtpReg, pThis->toWrkShutdown));
	CHKiRet(wtpSetpUsr		(pThis->pWtpReg, pThis));
	CHKiRet(wtpConstructFinalize	(pThis->pWtpReg));

	/* set up DA system if we have a disk-assisted queue */
	if(pThis->bIsDA)
		InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */

	DBGOPRINT((obj_t*) pThis, "queue finished initialization\n");

	/* if the queue already contains data, we need to start the correct number of worker threads. This can be
	 * the case when a disk queue has been loaded. If we did not start it here, it would never start.
	 */
	qqueueAdviseMaxWorkers(pThis);
	pThis->bQueueStarted = 1;

finalize_it:
	RETiRet;
}


static rsRetVal qDelLinkedList(qqueue_t *pThis)
{
	qLinkedList_t *pEntry;
	DEFiRet;

	pEntry = pThis->tVars.linklist.pDelRoot;

	if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) {
		pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL;
	} else {
		pThis->tVars.linklist.pDelRoot = pEntry->pNext;
	}

	free(pEntry);

	RETiRet;
}


	if(pThis->pqDA != NULL) {
		qqueueDestruct(&pThis->pqDA);
	}


GetDeqBatchSize(qqueue_t *pThis, int *pVal)
{
	DEFiRet;
	assert(pVal != NULL);
	*pVal = pThis->iDeqBatchSize;
if(pThis->pqParent != NULL) // TODO: check why we actually do this!
	*pVal = 16;
	RETiRet;
}


doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
{
	DEFiRet;
	struct timespec t;

	/* first check if we need to discard this message (which will cause CHKiRet() to exit)
	 */
	CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr));

	/* handle flow control
	 * There are two different flow control mechanisms: basic and advanced flow control.
	 * Basic flow control has always been implemented and protects the queue structures
	 * in that it makes sure no more data is enqueued than the queue is configured to
	 * support. Enhanced flow control is being added today. There are some sources which
	 * can easily be stopped, e.g. a file reader. This is the case because it is unlikely
	 * that blocking those sources will have negative effects (after all, the file is
	 * continued to be written). Other sources can somewhat be blocked (e.g. the kernel
	 * log reader or the local log stream reader): in general, nothing is lost if messages
	 * from these sources are not picked up immediately. HOWEVER, they can not block for
	 * an extended period of time, as this either causes message loss or - even worse - some
	 * other bad effects (e.g. unresponsive system in respect to the main system log socket).
	 * Finally, there are some (few) sources which can not be blocked at all. UDP syslog is
	 * a prime example. If a UDP message is not received, it is simply lost. So we can't
	 * do anything against UDP sockets that come in too fast. The core idea of advanced
	 * flow control is that we take into account the different natures of the sources and
	 * select flow control mechanisms that fit these needs. This also means, in the end
	 * result, that non-blockable sources like UDP syslog receive priority in the system.
	 * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14
	 */
	if(flowCtlType == eFLOWCTL_FULL_DELAY) {
		while(pThis->iQueueSize >= pThis->iFullDlyMrk) {
			DBGOPRINT((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n");
			pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */
		}
	} else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) {
		if(pThis->iQueueSize >= pThis->iLightDlyMrk) {
			DBGOPRINT((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n");
			timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */
			pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */
		}
	}

	/* from our regular flow control settings, we are now ready to enqueue the object.
	 * However, we now need to do a check if the queue permits to add more data. If that
	 * is not the case, basic flow control enters the field, which means we wait for
	 * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14
	 */
	while(   (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize)
	      || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0
	      	  && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) {
		DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n");
		timeoutComp(&t, pThis->toEnq);
// TODO : handle enqOnly => discard!
		if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) {
			DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n");
			objDestruct(pUsr);
			ABORT_FINALIZE(RS_RET_QUEUE_FULL);
		}
	}

	/* and finally enqueue the message */
	CHKiRet(qqueueAdd(pThis, pUsr));

finalize_it:
	RETiRet;
}


qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr)
{
	DEFiRet;
	int iCancelStateSave;

	ISOBJ_TYPE_assert(pThis, qqueue);

	if(pThis->qType != QUEUETYPE_DIRECT) {
		pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
		d_pthread_mutex_lock(pThis->mut);
	}

	CHKiRet(doEnqSingleObj(pThis, flowCtlType, pUsr));

	qqueueChkPersist(pThis, 1);

finalize_it:
	if(pThis->qType != QUEUETYPE_DIRECT) {
		/* make sure at least one worker is running. */
		qqueueAdviseMaxWorkers(pThis);
		/* and release the mutex */
		d_pthread_mutex_unlock(pThis->mut);
		pthread_setcancelstate(iCancelStateSave, NULL);
		DBGOPRINT((obj_t*) pThis, "EnqueueMsg advised worker start\n");
	}

	RETiRet;
}


DoSaveOnShutdown(qqueue_t *pThis)
{
	struct timespec tTimeout;
	rsRetVal iRetLocal;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);

	/* we reduce the low water mark, otherwise the DA worker would terminate when
	 * it is reached.
	 */
	DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n");
	pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */
	pThis->iLowWtrMrk = 0;
	wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN);	/* shutdown worker (only) when done (was _IMMEDIATE!) */
	wtpAdviseMaxWorkers(pThis->pWtpDA, 1);		/* restart DA worker */

	DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n");
	timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL);
	/* and run the primary queue's DA worker to drain the queue */
	iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout);
	DBGOPRINT((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n",
		  iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
	if(iRetLocal != RS_RET_OK) {
		DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, "
			  "continuing, but results are unpredictable\n", iRetLocal);
	}

	RETiRet;
}


qqueueAdd(qqueue_t *pThis, void *pUsr)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	CHKiRet(pThis->qAdd(pThis, pUsr));

	if(pThis->qType != QUEUETYPE_DIRECT) {
		ATOMIC_INC(&pThis->iQueueSize, &pThis->mutQueueSize);
		DBGOPRINT((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n",
			  getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis));
	}

finalize_it:
	RETiRet;
}


qqueueMultiEnqObjDirect(qqueue_t *pThis, multi_submit_t *pMultiSub)
{
	int i;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	assert(pMultiSub != NULL);

	for(i = 0 ; i < pMultiSub->nElem ; ++i) {
		CHKiRet(qAddDirect(pThis, (void*)pMultiSub->ppMsgs[i]));
	}

finalize_it:
	RETiRet;
}


batchProcessed(qqueue_t *pThis, wti_t *pWti)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ISOBJ_TYPE_assert(pWti, wti);

	int iCancelStateSave;
	/* at this spot, we must not be cancelled */
	pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave);
	DeleteProcessedBatch(pThis, &pWti->batch);
	qqueueChkPersist(pThis, pWti->batch.nElemDeq);
	pthread_setcancelstate(iCancelStateSave, NULL);

	RETiRet;
}


DequeueForConsumer(qqueue_t *pThis, wti_t *pWti)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ISOBJ_TYPE_assert(pWti, wti);

	CHKiRet(DequeueConsumable(pThis, pWti));

	if(pWti->batch.nElem == 0)
		ABORT_FINALIZE(RS_RET_IDLE);

finalize_it:
	RETiRet;
}


StartDA(qqueue_t *pThis)
{
	DEFiRet;
	uchar pszDAQName[128];

	ISOBJ_TYPE_assert(pThis, qqueue);

	/* create message queue */
	CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer));

	/* give it a name */
	snprintf((char*) pszDAQName, sizeof(pszDAQName)/sizeof(uchar), "%s[DA]", obj.GetName((obj_t*) pThis));
	obj.SetName((obj_t*) pThis->pqDA, pszDAQName);

	/* as the created queue is the same object class, we take the
	 * liberty to access its properties directly.
	 */
	pThis->pqDA->pqParent = pThis;

	CHKiRet(qqueueSetpUsr(pThis->pqDA, pThis->pUsr));
	CHKiRet(qqueueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax));
	CHKiRet(qqueueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown));
	CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize));
	CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix));
	CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt));
	CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles));
	CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown));
	CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq));
	CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr));
	CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr));
	CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown));
	CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0));
	CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0));

	iRet = qqueueStart(pThis->pqDA);
	/* file not found is expected, that means it is no previous QIF available */
	if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) {
		errno = 0; /* else an errno is shown in errmsg! */
		errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode");
		pThis->bIsDA = 0;	/* disable memory mode */
		FINALIZE; /* something is wrong */
	}

	DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n",
		  qqueueGetID(pThis->pqDA));

finalize_it:
	if(iRet != RS_RET_OK) {
		if(pThis->pqDA != NULL) {
			qqueueDestruct(&pThis->pqDA);
		}
		DBGOPRINT((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet);
		pThis->bIsDA = 0;
	}

	RETiRet;
}


static rsRetVal qDestructFixedArray(qqueue_t *pThis)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	queueDrain(pThis); /* discard any remaining queue entries */
	free(pThis->tVars.farray.pBuf);

	RETiRet;
}


static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out)
{
	DEFiRet;

	ASSERT(pThis != NULL);
	*out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead];

	pThis->tVars.farray.deqhead++;
	if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize)
		pThis->tVars.farray.deqhead = 0;

	RETiRet;
}


DequeueConsumable(qqueue_t *pThis, wti_t *pWti)
{
	DEFiRet;
	int iQueueSize = 0; /* keep the compiler happy... */

	/* dequeue element batch (still protected from mutex) */
	iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize);

	/* awake some flow-controlled sources if we can do this right now */
	/* TODO: this could be done better from a performance point of view -- do it only if
	 * we have someone waiting for the condition (or only when we hit the watermark right
	 * on the nail [exact value]) -- rgerhards, 2008-03-14
	 * now that we dequeue batches of pointers, this is much less an issue...
	 * rgerhards, 2009-04-22
	 */
	if(iQueueSize < pThis->iFullDlyMrk / 2) {
		pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk);
	}

	if(iQueueSize < pThis->iLightDlyMrk / 2) {
		pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk);
	}

	// TODO: MULTI: check physical queue size?
	pthread_cond_signal(&pThis->notFull);
	/* WE ARE NO LONGER PROTECTED BY THE MUTEX */

	if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) {
		DBGOPRINT((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things "
			  "may happen\n", iRet);
	}

	RETiRet;
}


DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch)
{
	int i;
	void *pUsr;
	int nEnqueued = 0;
	rsRetVal localRet;
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	assert(pBatch != NULL);

	for(i = 0 ; i < pBatch->nElem ; ++i) {
		pUsr = pBatch->pElem[i].pUsrp;
		if(   pBatch->pElem[i].state == BATCH_STATE_RDY
		   || pBatch->pElem[i].state == BATCH_STATE_SUB) {
dbgprintf("XXX: DeleteProcessedBatch re-enqueue %d of %d, state %d\n", i, pBatch->nElem, pBatch->pElem[i].state);
			localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY,
				       (obj_t*)MsgAddRef((msg_t*) pUsr));
			++nEnqueued;
			if(localRet != RS_RET_OK) {
				DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet);
			}
		}
		objDestruct(pUsr);
	}

	dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued);

	if(nEnqueued > 0)
		qqueueChkPersist(pThis, nEnqueued);

	iRet = DeleteBatchFromQStore(pThis, pBatch);

	pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ // TODO: more fine init, new fields! 2010-06-14

	RETiRet;
}


qqueueTryLoadPersistedInfo(qqueue_t *pThis)
{
	DEFiRet;
	strm_t *psQIF = NULL;
	uchar pszQIFNam[MAXFNAME];
	size_t lenQIFNam;
	struct stat stat_buf;

	ISOBJ_TYPE_assert(pThis, qqueue);

	/* Construct file name */
	lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi",
			     (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix);

	/* check if the file exists */
	if(stat((char*) pszQIFNam, &stat_buf) == -1) {
		if(errno == ENOENT) {
			DBGOPRINT((obj_t*) pThis, "clean startup, no .qi file found\n");
			ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND);
		} else {
			DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno);
			ABORT_FINALIZE(RS_RET_IO_ERROR);
		}
	}

	/* If we reach this point, we have a .qi file */

	CHKiRet(strm.Construct(&psQIF));
	CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ));
	CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE));
	CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam));
	CHKiRet(strm.ConstructFinalize(psQIF));

	/* first, we try to read the property bag for ourselfs */
	CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF));

	/* then the stream objects (same order as when persisted!) */
	CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF,
			       (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis));
	CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF,
			       (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis));

	/* create a duplicate for the read "pointer".
	 */

	CHKiRet(strm.Dup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq));
	CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */
	CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pReadDeq));

	CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite));
	CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDel));
	CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pReadDeq));

	/* OK, we could successfully read the file, so we now can request that it be
	 * deleted when we are done with the persisted information.
	 */
	pThis->bNeedDelQIF = 1;

finalize_it:
	if(psQIF != NULL)
		strm.Destruct(&psQIF);

	if(iRet != RS_RET_OK) {
		DBGOPRINT((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n",
			  iRet);
	}

	RETiRet;
}


static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ASSERT(pProp != NULL);

 	if(isProp("iQueueSize")) {
		pThis->iQueueSize = pProp->val.num;
 	} else if(isProp("tVars.disk.sizeOnDisk")) {
		pThis->tVars.disk.sizeOnDisk = pProp->val.num;
 	} else if(isProp("tVars.disk.bytesRead")) {
		pThis->tVars.disk.bytesRead = pProp->val.num;
 	} else if(isProp("qType")) {
		if(pThis->qType != pProp->val.num)
			ABORT_FINALIZE(RS_RET_QTYPE_MISMATCH);
	}

finalize_it:
	RETiRet;
}


DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize)
{
	int nDequeued;
	int nDiscarded;
	int nDeleted;
	int iQueueSize;
	void *pUsr;
	rsRetVal localRet;
	DEFiRet;

	nDeleted = pWti->batch.nElemDeq;
	DeleteProcessedBatch(pThis, &pWti->batch);

	nDequeued = nDiscarded = 0;
	while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) {
		CHKiRet(qqueueDeq(pThis, &pUsr));

		/* check if we should discard this element */
		localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr);
		if(localRet == RS_RET_QUEUE_FULL) {
			++nDiscarded;
			continue;
		} else if(localRet != RS_RET_OK) {
			ABORT_FINALIZE(localRet);
		}

		/* all well, use this element */
		pWti->batch.pElem[nDequeued].pUsrp = pUsr;
		pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY;
		pWti->batch.pElem[nDequeued].bFilterOK = 1; // TODO: think again if we can handle that with more performance
		++nDequeued;
	}

	/* it is sufficient to persist only when the bulk of work is done */
	qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted);

	pWti->batch.nElem = nDequeued;
	pWti->batch.nElemDeq = nDequeued + nDiscarded;
	pWti->batch.deqID = getNextDeqID(pThis);
	*piRemainingQueueSize = iQueueSize;

finalize_it:
	RETiRet;
}


static inline void queueDrain(qqueue_t *pThis)
{
	void *pUsr;
	ASSERT(pThis != NULL);

	BEGINfunc
	DBGOPRINT((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize);
	/* iQueueSize is not decremented by qDel(), so we need to do it ourselves */
	while(ATOMIC_DEC_AND_FETCH(&pThis->iQueueSize, &pThis->mutQueueSize) > 0) {
		pThis->qDeq(pThis, &pUsr);
		if(pUsr != NULL) {
			objDestruct(pUsr);
		}
		pThis->qDel(pThis);
	}
	ENDfunc
}


static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue)
{
	ISOBJ_TYPE_assert(pQueue, qqueue);
	return pQueue->toDeleteLst;
}


qqueueChkStopWrkrDA(qqueue_t *pThis)
{
	DEFiRet;

	if(pThis->bEnqOnly) {
		iRet = RS_RET_TERMINATE_WHEN_IDLE;
	}

	RETiRet;
}


static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr)
{
	DEFiRet;
	rsRetVal iRetLocal;
	int iSeverity;

	ISOBJ_TYPE_assert(pThis, qqueue);
	ISOBJ_assert(pUsr);

	if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) {
		iRetLocal = objGetSeverity(pUsr, &iSeverity);
		if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) {
			DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n",
				  iQueueSize, iSeverity);
			objDestruct(pUsr);
			ABORT_FINALIZE(RS_RET_QUEUE_FULL);
		} else {
			DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), but could not drop msg "
				  "(iRet: %d, severity %d)\n", iQueueSize, iRetLocal, iSeverity);
		}
	}

finalize_it:
	RETiRet;
}


qqueueChkIsDA(qqueue_t *pThis)
{
	DEFiRet;

	ISOBJ_TYPE_assert(pThis, qqueue);
	if(pThis->pszFilePrefix != NULL) {
		pThis->bIsDA = 1;
		DBGOPRINT((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n");
	} else {
		DBGOPRINT((obj_t*) pThis, "is NOT disk-assisted\n");
	}

	RETiRet;
}


	if(pThis->qType != QUEUETYPE_DIRECT && pThis->pWtpReg != NULL) {
		wtpDestruct(&pThis->pWtpReg);
	}


static rsRetVal qDestructDisk(qqueue_t *pThis)
{
	DEFiRet;

	ASSERT(pThis != NULL);

	if(pThis->tVars.disk.pWrite != NULL)
		strm.Destruct(&pThis->tVars.disk.pWrite);
	if(pThis->tVars.disk.pReadDeq != NULL)
		strm.Destruct(&pThis->tVars.disk.pReadDeq);
	if(pThis->tVars.disk.pReadDel != NULL)
		strm.Destruct(&pThis->tVars.disk.pReadDel);

	RETiRet;
}


static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis)
{
	return RS_RET_OK;
}


STATIC void GC_init_explicit_typing(void)
{
    unsigned i;

    GC_STATIC_ASSERT(sizeof(struct LeafDescriptor) % sizeof(word) == 0);
    /* Set up object kind with simple indirect descriptor. */
      GC_eobjfreelist = (ptr_t *)GC_new_free_list_inner();
      GC_explicit_kind = GC_new_kind_inner(
                            (void **)GC_eobjfreelist,
                            (WORDS_TO_BYTES((word)-1) | GC_DS_PER_OBJECT),
                            TRUE, TRUE);
                /* Descriptors are in the last word of the object. */
      GC_typed_mark_proc_index = GC_new_proc_inner(GC_typed_mark_proc);
    /* Set up object kind with array descriptor. */
      GC_array_mark_proc_index = GC_new_proc_inner(GC_array_mark_proc);
      GC_array_kind = GC_new_kind_inner(GC_new_free_list_inner(),
                            GC_MAKE_PROC(GC_array_mark_proc_index, 0),
                            FALSE, TRUE);
      GC_bm_table[0] = GC_DS_BITMAP;
      for (i = 1; i < WORDSZ/2; i++) {
          GC_bm_table[i] = (((word)-1) << (WORDSZ - i)) | GC_DS_BITMAP;
      }
}

Fix calloc_explicitly_typed in case of lb*n overflow

* typd_mlc.c: Include limits.h (for SIZE_MAX).
* typd_mlc.c (GC_SIZE_MAX, GC_SQRT_SIZE_MAX): New macro (same as in
malloc.c).
* typd_mlc.c (GC_calloc_explicitly_typed): Return NULL if lb * n
overflows (same algorithm as in calloc defined in malloc.c); eliminate
lb *= n code duplication.

#define TYPD_EXTRA_BYTES (sizeof(word) - EXTRA_BYTES)

STATIC int GC_explicit_kind = 0;

STATIC int GC_array_kind = 0;

typedef struct {
        word ed_bitmap; /* lsb corresponds to first word.       */
        GC_bool ed_continued;   /* next entry is continuation.  */
} ext_descr;

typedef union ComplexDescriptor {
    struct LeafDescriptor {     /* Describes simple array       */
        word ld_tag;
#       define LEAF_TAG 1
        size_t ld_size;         /* bytes per element    */
                                /* multiple of ALIGNMENT        */
        size_t ld_nelements;    /* Number of elements.  */
        GC_descr ld_descriptor; /* A simple length, bitmap,     */
                                /* or procedure descriptor.     */
    } ld;
    struct ComplexArrayDescriptor {
        word ad_tag;
#       define ARRAY_TAG 2
        size_t ad_nelements;
        union ComplexDescriptor * ad_element_descr;
    } ad;
    struct SequenceDescriptor {
        word sd_tag;
#       define SEQUENCE_TAG 3
        union ComplexDescriptor * sd_first;
        union ComplexDescriptor * sd_second;
    } sd;
} complex_descriptor;

STATIC ext_descr * GC_ext_descriptors = NULL;

STATIC size_t GC_ed_size = 0;

STATIC size_t GC_avail_descr = 0;

STATIC int GC_typed_mark_proc_index = 0;

STATIC int GC_array_mark_proc_index = 0;

STATIC GC_descr GC_bm_table[WORDSZ/2];

STATIC complex_descriptor

STATIC ptr_t * GC_eobjfreelist = NULL;

STATIC mse * GC_typed_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);

STATIC mse * GC_array_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);

#   define HIGH_BIT (((word)1) << (WORDSZ - 1))

#define GC_SQRT_SIZE_MAX ((1U << (WORDSZ / 2)) - 1)


GC_API GC_descr GC_CALL GC_make_descriptor(const GC_word * bm, size_t len)
{
    signed_word last_set_bit = len - 1;
    GC_descr result;
    signed_word i;
#   define HIGH_BIT (((word)1) << (WORDSZ - 1))
    DCL_LOCK_STATE;

#   if defined(THREADS) && defined(AO_HAVE_load_acquire)
      if (!EXPECT(AO_load_acquire(
                        (volatile AO_t *)&GC_explicit_typing_initialized),
                  TRUE))
#   endif
    {
      LOCK();
#     if defined(THREADS) && defined(AO_HAVE_load_acquire)
        if (!GC_explicit_typing_initialized)
#     else
        if (!EXPECT(GC_explicit_typing_initialized, TRUE))
#     endif
      {
        GC_init_explicit_typing();
        GC_explicit_typing_initialized = TRUE;
      }
      UNLOCK();
    }

    while (last_set_bit >= 0 && !GC_get_bit(bm, last_set_bit))
      last_set_bit--;
    if (last_set_bit < 0) return(0 /* no pointers */);
#   if ALIGNMENT == CPP_WORDSZ/8
    {
      for (i = 0; i < last_set_bit; i++) {
        if (!GC_get_bit(bm, i)) {
          break;
        }
      }
      if (i == last_set_bit) {
        /* An initial section contains all pointers.  Use length descriptor. */
        return (WORDS_TO_BYTES(last_set_bit+1) | GC_DS_LENGTH);
      }
    }
#   endif
    if ((word)last_set_bit < BITMAP_BITS) {
        /* Hopefully the common case.                   */
        /* Build bitmap descriptor (with bits reversed) */
        result = HIGH_BIT;
        for (i = last_set_bit - 1; i >= 0; i--) {
            result >>= 1;
            if (GC_get_bit(bm, i)) result |= HIGH_BIT;
        }
        result |= GC_DS_BITMAP;
    } else {
        signed_word index = GC_add_ext_descriptor(bm, (word)last_set_bit + 1);
        if (index == -1) return(WORDS_TO_BYTES(last_set_bit+1) | GC_DS_LENGTH);
                                /* Out of memory: use conservative      */
                                /* approximation.                       */
        result = GC_MAKE_PROC(GC_typed_mark_proc_index, (word)index);
    }
    return result;
}

#define TYPD_EXTRA_BYTES (sizeof(word) - EXTRA_BYTES)

typedef struct {
        word ed_bitmap; /* lsb corresponds to first word.       */
        GC_bool ed_continued;   /* next entry is continuation.  */
} ext_descr;

typedef union ComplexDescriptor {
    struct LeafDescriptor {     /* Describes simple array       */
        word ld_tag;
#       define LEAF_TAG 1
        size_t ld_size;         /* bytes per element    */
                                /* multiple of ALIGNMENT        */
        size_t ld_nelements;    /* Number of elements.  */
        GC_descr ld_descriptor; /* A simple length, bitmap,     */
                                /* or procedure descriptor.     */
    } ld;
    struct ComplexArrayDescriptor {
        word ad_tag;
#       define ARRAY_TAG 2
        size_t ad_nelements;
        union ComplexDescriptor * ad_element_descr;
    } ad;
    struct SequenceDescriptor {
        word sd_tag;
#       define SEQUENCE_TAG 3
        union ComplexDescriptor * sd_first;
        union ComplexDescriptor * sd_second;
    } sd;
} complex_descriptor;

#define TAG ld.ld_tag

STATIC size_t GC_ed_size = 0;

#define ED_INITIAL_SIZE 100

STATIC size_t GC_avail_descr = 0;

STATIC int GC_typed_mark_proc_index = 0;

STATIC GC_descr GC_bm_table[WORDSZ/2];

#define COMPLEX 2

#define LEAF    1

#define SIMPLE  0

#define NO_MEM  (-1)

STATIC mse * GC_typed_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);

STATIC mse * GC_array_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);

GC_API GC_descr

signed_word last_set_bit = len - 1;

GC_descr result;

signed_word i;

#   define HIGH_BIT (((word)1) << (WORDSZ - 1))

GC_API GC_ATTR_MALLOC

# define GC_SIZE_MAX SIZE_MAX

# define GC_SIZE_MAX (~(size_t)0)

#define GC_SQRT_SIZE_MAX ((1U << (WORDSZ / 2)) - 1)


STATIC void GC_push_typed_structures_proc(void)
{
  GC_PUSH_ALL_SYM(GC_ext_descriptors);
}

STATIC int GC_explicit_kind = 0;

STATIC int GC_array_kind = 0;

STATIC ext_descr * GC_ext_descriptors = NULL;

STATIC size_t GC_ed_size = 0;

STATIC size_t GC_avail_descr = 0;

STATIC int GC_typed_mark_proc_index = 0;

STATIC int GC_array_mark_proc_index = 0;

STATIC GC_descr GC_bm_table[WORDSZ/2];

STATIC complex_descriptor

STATIC ptr_t * GC_eobjfreelist = NULL;

STATIC mse * GC_typed_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);

STATIC mse * GC_array_mark_proc(word * addr, mse * mark_stack_ptr,
                                mse * mark_stack_limit, word env);