sys/dev/sound/pcm/feeder_rate.c

   1 /*-
   2  * Copyright (c) 2005-2009 Ariff Abdullah <ariff@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28  * feeder_rate: (Codename: Z Resampler), which means any effort to create
  29  *              future replacement for this resampler are simply absurd unless
  30  *              the world decide to add new alphabet after Z.
  31  *
  32  * FreeBSD bandlimited sinc interpolator, technically based on
  33  * "Digital Audio Resampling" by Julius O. Smith III
  34  *  - http://ccrma.stanford.edu/~jos/resample/
  35  *
  36  * The Good:
  37  * + all out fixed point integer operations, no soft-float or anything like
  38  *   that.
  39  * + classic polyphase converters with high quality coefficient's polynomial
  40  *   interpolators.
  41  * + fast, faster, or the fastest of its kind.
  42  * + compile time configurable.
  43  * + etc etc..
  44  *
  45  * The Bad:
  46  * - The z, z_, and Z_ . Due to mental block (or maybe just 0x7a69), I
  47  *   couldn't think of anything simpler than that (feeder_rate_xxx is just
  48  *   too long). Expect possible clashes with other zitizens (any?).
  49  */
  50
  51 #ifdef _KERNEL
  52 #ifdef HAVE_KERNEL_OPTION_HEADERS
  53 #include "opt_snd.h"
  54 #endif
  55 #include <dev/sound/pcm/sound.h>
  56 #include <dev/sound/pcm/pcm.h>
  57 #include "feeder_if.h"
  58
  59 #define SND_USE_FXDIV
  60 #include "snd_fxdiv_gen.h"
  61
  62 SND_DECLARE_FILE("$FreeBSD: head/sys/dev/sound/pcm/feeder_rate.c 267992 2014-06-28 03:56:17Z hselasky $");
  63 #endif
  64
  65 #include "feeder_rate_gen.h"
  66
  67 #if !defined(_KERNEL) && defined(SND_DIAGNOSTIC)
  68 #undef Z_DIAGNOSTIC
  69 #define Z_DIAGNOSTIC            1
  70 #elif defined(_KERNEL)
  71 #undef Z_DIAGNOSTIC
  72 #endif
  73
  74 #ifndef Z_QUALITY_DEFAULT
  75 #define Z_QUALITY_DEFAULT       Z_QUALITY_LINEAR
  76 #endif
  77
  78 #define Z_RESERVOIR             2048
  79 #define Z_RESERVOIR_MAX         131072
  80
  81 #define Z_SINC_MAX              0x3fffff
  82 #define Z_SINC_DOWNMAX          48              /* 384000 / 8000 */
  83
  84 #ifdef _KERNEL
  85 #define Z_POLYPHASE_MAX         183040          /* 286 taps, 640 phases */
  86 #else
  87 #define Z_POLYPHASE_MAX         1464320         /* 286 taps, 5120 phases */
  88 #endif
  89
  90 #define Z_RATE_DEFAULT          48000
  91
  92 #define Z_RATE_MIN              FEEDRATE_RATEMIN
  93 #define Z_RATE_MAX              FEEDRATE_RATEMAX
  94 #define Z_ROUNDHZ               FEEDRATE_ROUNDHZ
  95 #define Z_ROUNDHZ_MIN           FEEDRATE_ROUNDHZ_MIN
  96 #define Z_ROUNDHZ_MAX           FEEDRATE_ROUNDHZ_MAX
  97
  98 #define Z_RATE_SRC              FEEDRATE_SRC
  99 #define Z_RATE_DST              FEEDRATE_DST
 100 #define Z_RATE_QUALITY          FEEDRATE_QUALITY
 101 #define Z_RATE_CHANNELS         FEEDRATE_CHANNELS
 102
 103 #define Z_PARANOID              1
 104
 105 #define Z_MULTIFORMAT           1
 106
 107 #ifdef _KERNEL
 108 #undef Z_USE_ALPHADRIFT
 109 #define Z_USE_ALPHADRIFT        1
 110 #endif
 111
 112 #define Z_FACTOR_MIN            1
 113 #define Z_FACTOR_MAX            Z_MASK
 114 #define Z_FACTOR_SAFE(v)        (!((v) < Z_FACTOR_MIN || (v) > Z_FACTOR_MAX))
 115
 116 struct z_info;
 117
 118 typedef void (*z_resampler_t)(struct z_info *, uint8_t *);
 119
 120 struct z_info {
 121         int32_t rsrc, rdst;     /* original source / destination rates */
 122         int32_t src, dst;       /* rounded source / destination rates */
 123         int32_t channels;       /* total channels */
 124         int32_t bps;            /* bytes-per-sample */
 125         int32_t quality;        /* resampling quality */
 126
 127         int32_t z_gx, z_gy;     /* interpolation / decimation ratio */
 128         int32_t z_alpha;        /* output sample time phase / drift */
 129         uint8_t *z_delay;       /* FIR delay line / linear buffer */
 130         int32_t *z_coeff;       /* FIR coefficients */
 131         int32_t *z_dcoeff;      /* FIR coefficients differences */
 132         int32_t *z_pcoeff;      /* FIR polyphase coefficients */
 133         int32_t z_scale;        /* output scaling */
 134         int32_t z_dx;           /* input sample drift increment */
 135         int32_t z_dy;           /* output sample drift increment */
 136 #ifdef Z_USE_ALPHADRIFT
 137         int32_t z_alphadrift;   /* alpha drift rate */
 138         int32_t z_startdrift;   /* buffer start position drift rate */
 139 #endif
 140         int32_t z_mask;         /* delay line full length mask */
 141         int32_t z_size;         /* half width of FIR taps */
 142         int32_t z_full;         /* full size of delay line */
 143         int32_t z_alloc;        /* largest allocated full size of delay line */
 144         int32_t z_start;        /* buffer processing start position */
 145         int32_t z_pos;          /* current position for the next feed */
 146 #ifdef Z_DIAGNOSTIC
 147         uint32_t z_cycle;       /* output cycle, purely for statistical */
 148 #endif
 149         int32_t z_maxfeed;      /* maximum feed to avoid 32bit overflow */
 150
 151         z_resampler_t z_resample;
 152 };
 153
 154 int feeder_rate_min = Z_RATE_MIN;
 155 int feeder_rate_max = Z_RATE_MAX;
 156 int feeder_rate_round = Z_ROUNDHZ;
 157 int feeder_rate_quality = Z_QUALITY_DEFAULT;
 158
 159 static int feeder_rate_polyphase_max = Z_POLYPHASE_MAX;
 160
 161 #ifdef _KERNEL
 162 static char feeder_rate_presets[] = FEEDER_RATE_PRESETS;
 163 SYSCTL_STRING(_hw_snd, OID_AUTO, feeder_rate_presets, CTLFLAG_RD,
 164     &feeder_rate_presets, 0, "compile-time rate presets");
 165
 166 TUNABLE_INT("hw.snd.feeder_rate_min", &feeder_rate_min);
 167 TUNABLE_INT("hw.snd.feeder_rate_max", &feeder_rate_max);
 168 TUNABLE_INT("hw.snd.feeder_rate_round", &feeder_rate_round);
 169 TUNABLE_INT("hw.snd.feeder_rate_quality", &feeder_rate_quality);
 170
 171 TUNABLE_INT("hw.snd.feeder_rate_polyphase_max", &feeder_rate_polyphase_max);
 172 SYSCTL_INT(_hw_snd, OID_AUTO, feeder_rate_polyphase_max, CTLFLAG_RW,
 173     &feeder_rate_polyphase_max, 0, "maximum allowable polyphase entries");
 174
 175 static int
 176 sysctl_hw_snd_feeder_rate_min(SYSCTL_HANDLER_ARGS)
 177 {
 178         int err, val;
 179
 180         val = feeder_rate_min;
 181         err = sysctl_handle_int(oidp, &val, 0, req);
 182
 183         if (err != 0 || req->newptr == NULL || val == feeder_rate_min)
 184                 return (err);
 185
 186         if (!(Z_FACTOR_SAFE(val) && val < feeder_rate_max))
 187                 return (EINVAL);
 188
 189         feeder_rate_min = val;
 190
 191         return (0);
 192 }
 193 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_min, CTLTYPE_INT | CTLFLAG_RW,
 194     0, sizeof(int), sysctl_hw_snd_feeder_rate_min, "I",
 195     "minimum allowable rate");
 196
 197 static int
 198 sysctl_hw_snd_feeder_rate_max(SYSCTL_HANDLER_ARGS)
 199 {
 200         int err, val;
 201
 202         val = feeder_rate_max;
 203         err = sysctl_handle_int(oidp, &val, 0, req);
 204
 205         if (err != 0 || req->newptr == NULL || val == feeder_rate_max)
 206                 return (err);
 207
 208         if (!(Z_FACTOR_SAFE(val) && val > feeder_rate_min))
 209                 return (EINVAL);
 210
 211         feeder_rate_max = val;
 212
 213         return (0);
 214 }
 215 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_max, CTLTYPE_INT | CTLFLAG_RW,
 216     0, sizeof(int), sysctl_hw_snd_feeder_rate_max, "I",
 217     "maximum allowable rate");
 218
 219 static int
 220 sysctl_hw_snd_feeder_rate_round(SYSCTL_HANDLER_ARGS)
 221 {
 222         int err, val;
 223
 224         val = feeder_rate_round;
 225         err = sysctl_handle_int(oidp, &val, 0, req);
 226
 227         if (err != 0 || req->newptr == NULL || val == feeder_rate_round)
 228                 return (err);
 229
 230         if (val < Z_ROUNDHZ_MIN || val > Z_ROUNDHZ_MAX)
 231                 return (EINVAL);
 232
 233         feeder_rate_round = val - (val % Z_ROUNDHZ);
 234
 235         return (0);
 236 }
 237 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_round, CTLTYPE_INT | CTLFLAG_RW,
 238     0, sizeof(int), sysctl_hw_snd_feeder_rate_round, "I",
 239     "sample rate converter rounding threshold");
 240
 241 static int
 242 sysctl_hw_snd_feeder_rate_quality(SYSCTL_HANDLER_ARGS)
 243 {
 244         struct snddev_info *d;
 245         struct pcm_channel *c;
 246         struct pcm_feeder *f;
 247         int i, err, val;
 248
 249         val = feeder_rate_quality;
 250         err = sysctl_handle_int(oidp, &val, 0, req);
 251
 252         if (err != 0 || req->newptr == NULL || val == feeder_rate_quality)
 253                 return (err);
 254
 255         if (val < Z_QUALITY_MIN || val > Z_QUALITY_MAX)
 256                 return (EINVAL);
 257
 258         feeder_rate_quality = val;
 259
 260         /*
 261          * Traverse all available channels on each device and try to
 262          * set resampler quality if and only if it is exist as
 263          * part of feeder chains and the channel is idle.
 264          */
 265         for (i = 0; pcm_devclass != NULL &&
 266             i < devclass_get_maxunit(pcm_devclass); i++) {
 267                 d = devclass_get_softc(pcm_devclass, i);
 268                 if (!PCM_REGISTERED(d))
 269                         continue;
 270                 PCM_LOCK(d);
 271                 PCM_WAIT(d);
 272                 PCM_ACQUIRE(d);
 273                 CHN_FOREACH(c, d, channels.pcm) {
 274                         CHN_LOCK(c);
 275                         f = chn_findfeeder(c, FEEDER_RATE);
 276                         if (f == NULL || f->data == NULL || CHN_STARTED(c)) {
 277                                 CHN_UNLOCK(c);
 278                                 continue;
 279                         }
 280                         (void)FEEDER_SET(f, FEEDRATE_QUALITY, val);
 281                         CHN_UNLOCK(c);
 282                 }
 283                 PCM_RELEASE(d);
 284                 PCM_UNLOCK(d);
 285         }
 286
 287         return (0);
 288 }
 289 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_quality, CTLTYPE_INT | CTLFLAG_RW,
 290     0, sizeof(int), sysctl_hw_snd_feeder_rate_quality, "I",
 291     "sample rate converter quality ("__XSTRING(Z_QUALITY_MIN)"=low .. "
 292     __XSTRING(Z_QUALITY_MAX)"=high)");
 293 #endif  /* _KERNEL */
 294
 295
 296 /*
 297  * Resampler type.
 298  */
 299 #define Z_IS_ZOH(i)             ((i)->quality == Z_QUALITY_ZOH)
 300 #define Z_IS_LINEAR(i)          ((i)->quality == Z_QUALITY_LINEAR)
 301 #define Z_IS_SINC(i)            ((i)->quality > Z_QUALITY_LINEAR)
 302
 303 /*
 304  * Macroses for accurate sample time drift calculations.
 305  *
 306  * gy2gx : given the amount of output, return the _exact_ required amount of
 307  *         input.
 308  * gx2gy : given the amount of input, return the _maximum_ amount of output
 309  *         that will be generated.
 310  * drift : given the amount of input and output, return the elapsed
 311  *         sample-time.
 312  */
 313 #define _Z_GCAST(x)             ((uint64_t)(x))
 314
 315 #ifdef __x86_64__
 316 #define Z_DIV(x, y)             ((x) / (y))
 317 #endif
 318
 319 #define _Z_GY2GX(i, a, v)                                               \
 320         Z_DIV(((_Z_GCAST((i)->z_gx) * (v)) + ((i)->z_gy - (a) - 1)),    \
 321         (i)->z_gy)
 322
 323 #define _Z_GX2GY(i, a, v)                                               \
 324         Z_DIV(((_Z_GCAST((i)->z_gy) * (v)) + (a)), (i)->z_gx)
 325
 326 #define _Z_DRIFT(i, x, y)                                               \
 327         ((_Z_GCAST((i)->z_gy) * (x)) - (_Z_GCAST((i)->z_gx) * (y)))
 328
 329 #define z_gy2gx(i, v)           _Z_GY2GX(i, (i)->z_alpha, v)
 330 #define z_gx2gy(i, v)           _Z_GX2GY(i, (i)->z_alpha, v)
 331 #define z_drift(i, x, y)        _Z_DRIFT(i, x, y)
 332
 333 /*
 334  * Macroses for SINC coefficients table manipulations.. whatever.
 335  */
 336 #define Z_SINC_COEFF_IDX(i)     ((i)->quality - Z_QUALITY_LINEAR - 1)
 337
 338 #define Z_SINC_LEN(i)                                                   \
 339         ((int32_t)(((uint64_t)z_coeff_tab[Z_SINC_COEFF_IDX(i)].len <<   \
 340             Z_SHIFT) / (i)->z_dy))
 341
 342 #define Z_SINC_BASE_LEN(i)                                              \
 343         ((z_coeff_tab[Z_SINC_COEFF_IDX(i)].len - 1) >> (Z_DRIFT_SHIFT - 1))
 344
 345 /*
 346  * Macroses for linear delay buffer operations. Alignment is not
 347  * really necessary since we're not using true circular buffer, but it
 348  * will help us guard against possible trespasser. To be honest,
 349  * the linear block operations does not need guarding at all due to
 350  * accurate drifting!
 351  */
 352 #define z_align(i, v)           ((v) & (i)->z_mask)
 353 #define z_next(i, o, v)         z_align(i, (o) + (v))
 354 #define z_prev(i, o, v)         z_align(i, (o) - (v))
 355 #define z_fetched(i)            (z_align(i, (i)->z_pos - (i)->z_start) - 1)
 356 #define z_free(i)               ((i)->z_full - (i)->z_pos)
 357
 358 /*
 359  * Macroses for Bla Bla .. :)
 360  */
 361 #define z_copy(src, dst, sz)    (void)memcpy(dst, src, sz)
 362 #define z_feed(...)             FEEDER_FEED(__VA_ARGS__)
 363
 364 static __inline uint32_t
 365 z_min(uint32_t x, uint32_t y)
 366 {
 367
 368         return ((x < y) ? x : y);
 369 }
 370
 371 static int32_t
 372 z_gcd(int32_t x, int32_t y)
 373 {
 374         int32_t w;
 375
 376         while (y != 0) {
 377                 w = x % y;
 378                 x = y;
 379                 y = w;
 380         }
 381
 382         return (x);
 383 }
 384
 385 static int32_t
 386 z_roundpow2(int32_t v)
 387 {
 388         int32_t i;
 389
 390         i = 1;
 391
 392         /*
 393          * Let it overflow at will..
 394          */
 395         while (i > 0 && i < v)
 396                 i <<= 1;
 397
 398         return (i);
 399 }
 400
 401 /*
 402  * Zero Order Hold, the worst of the worst, an insult against quality,
 403  * but super fast.
 404  */
 405 static void
 406 z_feed_zoh(struct z_info *info, uint8_t *dst)
 407 {
 408 #if 0
 409         z_copy(info->z_delay +
 410             (info->z_start * info->channels * info->bps), dst,
 411             info->channels * info->bps);
 412 #else
 413         uint32_t cnt;
 414         uint8_t *src;
 415
 416         cnt = info->channels * info->bps;
 417         src = info->z_delay + (info->z_start * cnt);
 418
 419         /*
 420          * This is a bit faster than doing bcopy() since we're dealing
 421          * with possible unaligned samples.
 422          */
 423         do {
 424                 *dst++ = *src++;
 425         } while (--cnt != 0);
 426 #endif
 427 }
 428
 429 /*
 430  * Linear Interpolation. This at least sounds better (perceptually) and fast,
 431  * but without any proper filtering which means aliasing still exist and
 432  * could become worst with a right sample. Interpolation centered within
 433  * Z_LINEAR_ONE between the present and previous sample and everything is
 434  * done with simple 32bit scaling arithmetic.
 435  */
 436 #define Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                                     \
 437 static void                                                                     \
 438 z_feed_linear_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)            \
 439 {                                                                               \
 440         int32_t z;                                                              \
 441         intpcm_t x, y;                                                          \
 442         uint32_t ch;                                                            \
 443         uint8_t *sx, *sy;                                                       \
 444                                                                                 \
 445         z = ((uint32_t)info->z_alpha * info->z_dx) >> Z_LINEAR_UNSHIFT;         \
 446                                                                                 \
 447         sx = info->z_delay + (info->z_start * info->channels *                  \
 448             PCM_##BIT##_BPS);                                                   \
 449         sy = sx - (info->channels * PCM_##BIT##_BPS);                           \
 450                                                                                 \
 451         ch = info->channels;                                                    \
 452                                                                                 \
 453         do {                                                                    \
 454                 x = _PCM_READ_##SIGN##BIT##_##ENDIAN(sx);                       \
 455                 y = _PCM_READ_##SIGN##BIT##_##ENDIAN(sy);                       \
 456                 x = Z_LINEAR_INTERPOLATE_##BIT(z, x, y);                        \
 457                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, x);                      \
 458                 sx += PCM_##BIT##_BPS;                                          \
 459                 sy += PCM_##BIT##_BPS;                                          \
 460                 dst += PCM_##BIT##_BPS;                                         \
 461         } while (--ch != 0);                                                    \
 462 }
 463
 464 /*
 465  * Userland clipping diagnostic check, not enabled in kernel compilation.
 466  * While doing sinc interpolation, unrealistic samples like full scale sine
 467  * wav will clip, but for other things this will not make any noise at all.
 468  * Everybody should learn how to normalized perceived loudness of their own
 469  * music/sounds/samples (hint: ReplayGain).
 470  */
 471 #ifdef Z_DIAGNOSTIC
 472 #define Z_CLIP_CHECK(v, BIT)    do {                                    \
 473         if ((v) > PCM_S##BIT##_MAX) {                                   \
 474                 fprintf(stderr, "Overflow: v=%jd, max=%jd\n",           \
 475                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MAX);         \
 476         } else if ((v) < PCM_S##BIT##_MIN) {                            \
 477                 fprintf(stderr, "Underflow: v=%jd, min=%jd\n",          \
 478                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MIN);         \
 479         }                                                               \
 480 } while (0)
 481 #else
 482 #define Z_CLIP_CHECK(...)
 483 #endif
 484
 485 #define Z_CLAMP(v, BIT)                                                 \
 486         (((v) > PCM_S##BIT##_MAX) ? PCM_S##BIT##_MAX :                  \
 487         (((v) < PCM_S##BIT##_MIN) ? PCM_S##BIT##_MIN : (v)))
 488
 489 /*
 490  * Sine Cardinal (SINC) Interpolation. Scaling is done in 64 bit, so
 491  * there's no point to hold the plate any longer. All samples will be
 492  * shifted to a full 32 bit, scaled and restored during write for
 493  * maximum dynamic range (only for downsampling).
 494  */
 495 #define _Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, adv)                      \
 496         c += z >> Z_SHIFT;                                              \
 497         z &= Z_MASK;                                                    \
 498         coeff = Z_COEFF_INTERPOLATE(z, z_coeff[c], z_dcoeff[c]);        \
 499         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                        \
 500         v += Z_NORM_##BIT((intpcm64_t)x * coeff);                       \
 501         z += info->z_dy;                                                \
 502         p adv##= info->channels * PCM_##BIT##_BPS
 503
 504 /*
 505  * XXX GCC4 optimization is such a !@#$%, need manual unrolling.
 506  */
 507 #if defined(__GNUC__) && __GNUC__ >= 4
 508 #define Z_SINC_ACCUMULATE(...)  do {                                    \
 509         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 510         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 511 } while (0)
 512 #define Z_SINC_ACCUMULATE_DECR          2
 513 #else
 514 #define Z_SINC_ACCUMULATE(...)  do {                                    \
 515         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
 516 } while (0)
 517 #define Z_SINC_ACCUMULATE_DECR          1
 518 #endif
 519
 520 #define Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                                       \
 521 static void                                                                     \
 522 z_feed_sinc_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)              \
 523 {                                                                               \
 524         intpcm64_t v;                                                           \
 525         intpcm_t x;                                                             \
 526         uint8_t *p;                                                             \
 527         int32_t coeff, z, *z_coeff, *z_dcoeff;                                  \
 528         uint32_t c, center, ch, i;                                              \
 529                                                                                 \
 530         z_coeff = info->z_coeff;                                                \
 531         z_dcoeff = info->z_dcoeff;                                              \
 532         center = z_prev(info, info->z_start, info->z_size);                     \
 533         ch = info->channels * PCM_##BIT##_BPS;                                  \
 534         dst += ch;                                                              \
 535                                                                                 \
 536         do {                                                                    \
 537                 dst -= PCM_##BIT##_BPS;                                         \
 538                 ch -= PCM_##BIT##_BPS;                                          \
 539                 v = 0;                                                          \
 540                 z = info->z_alpha * info->z_dx;                                 \
 541                 c = 0;                                                          \
 542                 p = info->z_delay + (z_next(info, center, 1) *                  \
 543                     info->channels * PCM_##BIT##_BPS) + ch;                     \
 544                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
 545                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, +);                \
 546                 z = info->z_dy - (info->z_alpha * info->z_dx);                  \
 547                 c = 0;                                                          \
 548                 p = info->z_delay + (center * info->channels *                  \
 549                     PCM_##BIT##_BPS) + ch;                                      \
 550                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
 551                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, -);                \
 552                 if (info->z_scale != Z_ONE)                                     \
 553                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
 554                 else                                                            \
 555                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
 556                 Z_CLIP_CHECK(v, BIT);                                           \
 557                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
 558         } while (ch != 0);                                                      \
 559 }
 560
 561 #define Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)                             \
 562 static void                                                                     \
 563 z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)    \
 564 {                                                                               \
 565         intpcm64_t v;                                                           \
 566         intpcm_t x;                                                             \
 567         uint8_t *p;                                                             \
 568         int32_t ch, i, start, *z_pcoeff;                                        \
 569                                                                                 \
 570         ch = info->channels * PCM_##BIT##_BPS;                                  \
 571         dst += ch;                                                              \
 572         start = z_prev(info, info->z_start, (info->z_size << 1) - 1) * ch;      \
 573                                                                                 \
 574         do {                                                                    \
 575                 dst -= PCM_##BIT##_BPS;                                         \
 576                 ch -= PCM_##BIT##_BPS;                                          \
 577                 v = 0;                                                          \
 578                 p = info->z_delay + start + ch;                                 \
 579                 z_pcoeff = info->z_pcoeff +                                     \
 580                     ((info->z_alpha * info->z_size) << 1);                      \
 581                 for (i = info->z_size; i != 0; i--) {                           \
 582                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
 583                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
 584                         z_pcoeff++;                                             \
 585                         p += info->channels * PCM_##BIT##_BPS;                  \
 586                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
 587                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
 588                         z_pcoeff++;                                             \
 589                         p += info->channels * PCM_##BIT##_BPS;                  \
 590                 }                                                               \
 591                 if (info->z_scale != Z_ONE)                                     \
 592                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
 593                 else                                                            \
 594                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
 595                 Z_CLIP_CHECK(v, BIT);                                           \
 596                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
 597         } while (ch != 0);                                                      \
 598 }
 599
 600 #define Z_DECLARE(SIGN, BIT, ENDIAN)                                    \
 601         Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                             \
 602         Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                               \
 603         Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)
 604
 605 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 606 Z_DECLARE(S, 16, LE)
 607 Z_DECLARE(S, 32, LE)
 608 #endif
 609 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 610 Z_DECLARE(S, 16, BE)
 611 Z_DECLARE(S, 32, BE)
 612 #endif
 613 #ifdef SND_FEEDER_MULTIFORMAT
 614 Z_DECLARE(S,  8, NE)
 615 Z_DECLARE(S, 24, LE)
 616 Z_DECLARE(S, 24, BE)
 617 Z_DECLARE(U,  8, NE)
 618 Z_DECLARE(U, 16, LE)
 619 Z_DECLARE(U, 24, LE)
 620 Z_DECLARE(U, 32, LE)
 621 Z_DECLARE(U, 16, BE)
 622 Z_DECLARE(U, 24, BE)
 623 Z_DECLARE(U, 32, BE)
 624 #endif
 625
 626 enum {
 627         Z_RESAMPLER_ZOH,
 628         Z_RESAMPLER_LINEAR,
 629         Z_RESAMPLER_SINC,
 630         Z_RESAMPLER_SINC_POLYPHASE,
 631         Z_RESAMPLER_LAST
 632 };
 633
 634 #define Z_RESAMPLER_IDX(i)                                              \
 635         (Z_IS_SINC(i) ? Z_RESAMPLER_SINC : (i)->quality)
 636
 637 #define Z_RESAMPLER_ENTRY(SIGN, BIT, ENDIAN)                                    \
 638         {                                                                       \
 639             AFMT_##SIGN##BIT##_##ENDIAN,                                        \
 640             {                                                                   \
 641                 [Z_RESAMPLER_ZOH]    = z_feed_zoh,                              \
 642                 [Z_RESAMPLER_LINEAR] = z_feed_linear_##SIGN##BIT##ENDIAN,       \
 643                 [Z_RESAMPLER_SINC]   = z_feed_sinc_##SIGN##BIT##ENDIAN,         \
 644                 [Z_RESAMPLER_SINC_POLYPHASE]   =                                \
 645                     z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN                   \
 646             }                                                                   \
 647         }
 648
 649 static const struct {
 650         uint32_t format;
 651         z_resampler_t resampler[Z_RESAMPLER_LAST];
 652 } z_resampler_tab[] = {
 653 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 654         Z_RESAMPLER_ENTRY(S, 16, LE),
 655         Z_RESAMPLER_ENTRY(S, 32, LE),
 656 #endif
 657 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
 658         Z_RESAMPLER_ENTRY(S, 16, BE),
 659         Z_RESAMPLER_ENTRY(S, 32, BE),
 660 #endif
 661 #ifdef SND_FEEDER_MULTIFORMAT
 662         Z_RESAMPLER_ENTRY(S,  8, NE),
 663         Z_RESAMPLER_ENTRY(S, 24, LE),
 664         Z_RESAMPLER_ENTRY(S, 24, BE),
 665         Z_RESAMPLER_ENTRY(U,  8, NE),
 666         Z_RESAMPLER_ENTRY(U, 16, LE),
 667         Z_RESAMPLER_ENTRY(U, 24, LE),
 668         Z_RESAMPLER_ENTRY(U, 32, LE),
 669         Z_RESAMPLER_ENTRY(U, 16, BE),
 670         Z_RESAMPLER_ENTRY(U, 24, BE),
 671         Z_RESAMPLER_ENTRY(U, 32, BE),
 672 #endif
 673 };
 674
 675 #define Z_RESAMPLER_TAB_SIZE                                            \
 676         ((int32_t)(sizeof(z_resampler_tab) / sizeof(z_resampler_tab[0])))
 677
 678 static void
 679 z_resampler_reset(struct z_info *info)
 680 {
 681
 682         info->src = info->rsrc - (info->rsrc % ((feeder_rate_round > 0 &&
 683             info->rsrc > feeder_rate_round) ? feeder_rate_round : 1));
 684         info->dst = info->rdst - (info->rdst % ((feeder_rate_round > 0 &&
 685             info->rdst > feeder_rate_round) ? feeder_rate_round : 1));
 686         info->z_gx = 1;
 687         info->z_gy = 1;
 688         info->z_alpha = 0;
 689         info->z_resample = NULL;
 690         info->z_size = 1;
 691         info->z_coeff = NULL;
 692         info->z_dcoeff = NULL;
 693         if (info->z_pcoeff != NULL) {
 694                 kfree(info->z_pcoeff, M_DEVBUF);
 695                 info->z_pcoeff = NULL;
 696         }
 697         info->z_scale = Z_ONE;
 698         info->z_dx = Z_FULL_ONE;
 699         info->z_dy = Z_FULL_ONE;
 700 #ifdef Z_DIAGNOSTIC
 701         info->z_cycle = 0;
 702 #endif
 703         if (info->quality < Z_QUALITY_MIN)
 704                 info->quality = Z_QUALITY_MIN;
 705         else if (info->quality > Z_QUALITY_MAX)
 706                 info->quality = Z_QUALITY_MAX;
 707 }
 708
 709 #ifdef Z_PARANOID
 710 static int32_t
 711 z_resampler_sinc_len(struct z_info *info)
 712 {
 713         int32_t c, z, len, lmax;
 714
 715         if (!Z_IS_SINC(info))
 716                 return (1);
 717
 718         /*
 719          * A rather careful (or useless) way to calculate filter length.
 720          * Z_SINC_LEN() itself is accurate enough to do its job. Extra
 721          * sanity checking is not going to hurt though..
 722          */
 723         c = 0;
 724         z = info->z_dy;
 725         len = 0;
 726         lmax = z_coeff_tab[Z_SINC_COEFF_IDX(info)].len;
 727
 728         do {
 729                 c += z >> Z_SHIFT;
 730                 z &= Z_MASK;
 731                 z += info->z_dy;
 732         } while (c < lmax && ++len > 0);
 733
 734         if (len != Z_SINC_LEN(info)) {
 735 #ifdef _KERNEL
 736                 kprintf("%s(): sinc l=%d != Z_SINC_LEN=%d\n",
 737                     __func__, len, Z_SINC_LEN(info));
 738 #else
 739                 fprintf(stderr, "%s(): sinc l=%d != Z_SINC_LEN=%d\n",
 740                     __func__, len, Z_SINC_LEN(info));
 741                 return (-1);
 742 #endif
 743         }
 744
 745         return (len);
 746 }
 747 #else
 748 #define z_resampler_sinc_len(i)         (Z_IS_SINC(i) ? Z_SINC_LEN(i) : 1)
 749 #endif
 750
 751 #define Z_POLYPHASE_COEFF_SHIFT         0
 752
 753 /*
 754  * Pick suitable polynomial interpolators based on filter oversampled ratio
 755  * (2 ^ Z_DRIFT_SHIFT).
 756  */
 757 #if !(defined(Z_COEFF_INTERP_ZOH) || defined(Z_COEFF_INTERP_LINEAR) ||          \
 758     defined(Z_COEFF_INTERP_QUADRATIC) || defined(Z_COEFF_INTERP_HERMITE) ||     \
 759     defined(Z_COEFF_INTER_BSPLINE) || defined(Z_COEFF_INTERP_OPT32X) ||         \
 760     defined(Z_COEFF_INTERP_OPT16X) || defined(Z_COEFF_INTERP_OPT8X) ||          \
 761     defined(Z_COEFF_INTERP_OPT4X) || defined(Z_COEFF_INTERP_OPT2X))
 762 #if Z_DRIFT_SHIFT >= 6
 763 #define Z_COEFF_INTERP_BSPLINE          1
 764 #elif Z_DRIFT_SHIFT >= 5
 765 #define Z_COEFF_INTERP_OPT32X           1
 766 #elif Z_DRIFT_SHIFT == 4
 767 #define Z_COEFF_INTERP_OPT16X           1
 768 #elif Z_DRIFT_SHIFT == 3
 769 #define Z_COEFF_INTERP_OPT8X            1
 770 #elif Z_DRIFT_SHIFT == 2
 771 #define Z_COEFF_INTERP_OPT4X            1
 772 #elif Z_DRIFT_SHIFT == 1
 773 #define Z_COEFF_INTERP_OPT2X            1
 774 #else
 775 #error "Z_DRIFT_SHIFT screwed!"
 776 #endif
 777 #endif
 778
 779 /*
 780  * In classic polyphase mode, the actual coefficients for each phases need to
 781  * be calculated based on default prototype filters. For highly oversampled
 782  * filter, linear or quadradatic interpolator should be enough. Anything less
 783  * than that require 'special' interpolators to reduce interpolation errors.
 784  *
 785  * "Polynomial Interpolators for High-Quality Resampling of Oversampled Audio"
 786  *    by Olli Niemitalo
 787  *    - http://www.student.oulu.fi/~oniemita/dsp/deip.pdf
 788  *
 789  */
 790 static int32_t
 791 z_coeff_interpolate(int32_t z, int32_t *z_coeff)
 792 {
 793         int32_t coeff;
 794 #if defined(Z_COEFF_INTERP_ZOH)
 795
 796         /* 1-point, 0th-order (Zero Order Hold) */
 797         z = z;
 798         coeff = z_coeff[0];
 799 #elif defined(Z_COEFF_INTERP_LINEAR)
 800         int32_t zl0, zl1;
 801
 802         /* 2-point, 1st-order Linear */
 803         zl0 = z_coeff[0];
 804         zl1 = z_coeff[1] - z_coeff[0];
 805
 806         coeff = Z_RSHIFT((int64_t)zl1 * z, Z_SHIFT) + zl0;
 807 #elif defined(Z_COEFF_INTERP_QUADRATIC)
 808         int32_t zq0, zq1, zq2;
 809
 810         /* 3-point, 2nd-order Quadratic */
 811         zq0 = z_coeff[0];
 812         zq1 = z_coeff[1] - z_coeff[-1];
 813         zq2 = z_coeff[1] + z_coeff[-1] - (z_coeff[0] << 1);
 814
 815         coeff = Z_RSHIFT((Z_RSHIFT((int64_t)zq2 * z, Z_SHIFT) +
 816             zq1) * z, Z_SHIFT + 1) + zq0;
 817 #elif defined(Z_COEFF_INTERP_HERMITE)
 818         int32_t zh0, zh1, zh2, zh3;
 819
 820         /* 4-point, 3rd-order Hermite */
 821         zh0 = z_coeff[0];
 822         zh1 = z_coeff[1] - z_coeff[-1];
 823         zh2 = (z_coeff[-1] << 1) - (z_coeff[0] * 5) + (z_coeff[1] << 2) -
 824             z_coeff[2];
 825         zh3 = z_coeff[2] - z_coeff[-1] + ((z_coeff[0] - z_coeff[1]) * 3);
 826
 827         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zh3 * z, Z_SHIFT) +
 828             zh2) * z, Z_SHIFT) + zh1) * z, Z_SHIFT + 1) + zh0;
 829 #elif defined(Z_COEFF_INTERP_BSPLINE)
 830         int32_t zb0, zb1, zb2, zb3;
 831
 832         /* 4-point, 3rd-order B-Spline */
 833         zb0 = Z_RSHIFT(0x15555555LL * (((int64_t)z_coeff[0] << 2) +
 834             z_coeff[-1] + z_coeff[1]), 30);
 835         zb1 = z_coeff[1] - z_coeff[-1];
 836         zb2 = z_coeff[-1] + z_coeff[1] - (z_coeff[0] << 1);
 837         zb3 = Z_RSHIFT(0x15555555LL * (((z_coeff[0] - z_coeff[1]) * 3) +
 838             z_coeff[2] - z_coeff[-1]), 30);
 839
 840         coeff = (Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zb3 * z, Z_SHIFT) +
 841             zb2) * z, Z_SHIFT) + zb1) * z, Z_SHIFT) + zb0 + 1) >> 1;
 842 #elif defined(Z_COEFF_INTERP_OPT32X)
 843         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 844         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 845
 846         /* 6-point, 5th-order Optimal 32x */
 847         zoz = z - (Z_ONE >> 1);
 848         zoe1 = z_coeff[1] + z_coeff[0];
 849         zoe2 = z_coeff[2] + z_coeff[-1];
 850         zoe3 = z_coeff[3] + z_coeff[-2];
 851         zoo1 = z_coeff[1] - z_coeff[0];
 852         zoo2 = z_coeff[2] - z_coeff[-1];
 853         zoo3 = z_coeff[3] - z_coeff[-2];
 854
 855         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
 856             (0x00170c29LL * zoe3), 30);
 857         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
 858             (0x008cd4dcLL * zoo3), 30);
 859         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
 860             (0x0160b5d0LL * zoe3), 30);
 861         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
 862             (0x01cfe914LL * zoo3), 30);
 863         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
 864             (0x015508ddLL * zoe3), 30);
 865         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
 866             (0x0082d81aLL * zoo3), 30);
 867
 868         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 869             (int64_t)zoc5 * zoz, Z_SHIFT) +
 870             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 871             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 872 #elif defined(Z_COEFF_INTERP_OPT16X)
 873         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 874         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 875
 876         /* 6-point, 5th-order Optimal 16x */
 877         zoz = z - (Z_ONE >> 1);
 878         zoe1 = z_coeff[1] + z_coeff[0];
 879         zoe2 = z_coeff[2] + z_coeff[-1];
 880         zoe3 = z_coeff[3] + z_coeff[-2];
 881         zoo1 = z_coeff[1] - z_coeff[0];
 882         zoo2 = z_coeff[2] - z_coeff[-1];
 883         zoo3 = z_coeff[3] - z_coeff[-2];
 884
 885         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
 886             (0x00170c29LL * zoe3), 30);
 887         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
 888             (0x008cd4dcLL * zoo3), 30);
 889         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
 890             (0x0160b5d0LL * zoe3), 30);
 891         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
 892             (0x01cfe914LL * zoo3), 30);
 893         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
 894             (0x015508ddLL * zoe3), 30);
 895         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
 896             (0x0082d81aLL * zoo3), 30);
 897
 898         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 899             (int64_t)zoc5 * zoz, Z_SHIFT) +
 900             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 901             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 902 #elif defined(Z_COEFF_INTERP_OPT8X)
 903         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 904         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 905
 906         /* 6-point, 5th-order Optimal 8x */
 907         zoz = z - (Z_ONE >> 1);
 908         zoe1 = z_coeff[1] + z_coeff[0];
 909         zoe2 = z_coeff[2] + z_coeff[-1];
 910         zoe3 = z_coeff[3] + z_coeff[-2];
 911         zoo1 = z_coeff[1] - z_coeff[0];
 912         zoo2 = z_coeff[2] - z_coeff[-1];
 913         zoo3 = z_coeff[3] - z_coeff[-2];
 914
 915         zoc0 = Z_RSHIFT((0x1aa9b47dLL * zoe1) + (0x053d9944LL * zoe2) +
 916             (0x0018b23fLL * zoe3), 30);
 917         zoc1 = Z_RSHIFT((0x14a104d1LL * zoo1) + (0x0d7d2504LL * zoo2) +
 918             (0x0094b599LL * zoo3), 30);
 919         zoc2 = Z_RSHIFT((-0x0d22530bLL * zoe1) + (0x0bb37a2cLL * zoe2) +
 920             (0x016ed8e0LL * zoe3), 30);
 921         zoc3 = Z_RSHIFT((-0x0d744b1cLL * zoo1) + (0x01649591LL * zoo2) +
 922             (0x01dae93aLL * zoo3), 30);
 923         zoc4 = Z_RSHIFT((0x02a7ee1bLL * zoe1) + (-0x03fbdb24LL * zoe2) +
 924             (0x0153ed07LL * zoe3), 30);
 925         zoc5 = Z_RSHIFT((0x04cf9b6cLL * zoo1) + (-0x0266b378LL * zoo2) +
 926             (0x007a7c26LL * zoo3), 30);
 927
 928         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 929             (int64_t)zoc5 * zoz, Z_SHIFT) +
 930             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 931             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 932 #elif defined(Z_COEFF_INTERP_OPT4X)
 933         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 934         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 935
 936         /* 6-point, 5th-order Optimal 4x */
 937         zoz = z - (Z_ONE >> 1);
 938         zoe1 = z_coeff[1] + z_coeff[0];
 939         zoe2 = z_coeff[2] + z_coeff[-1];
 940         zoe3 = z_coeff[3] + z_coeff[-2];
 941         zoo1 = z_coeff[1] - z_coeff[0];
 942         zoo2 = z_coeff[2] - z_coeff[-1];
 943         zoo3 = z_coeff[3] - z_coeff[-2];
 944
 945         zoc0 = Z_RSHIFT((0x1a8eda43LL * zoe1) + (0x0556ee38LL * zoe2) +
 946             (0x001a3784LL * zoe3), 30);
 947         zoc1 = Z_RSHIFT((0x143d863eLL * zoo1) + (0x0d910e36LL * zoo2) +
 948             (0x009ca889LL * zoo3), 30);
 949         zoc2 = Z_RSHIFT((-0x0d026821LL * zoe1) + (0x0b837773LL * zoe2) +
 950             (0x017ef0c6LL * zoe3), 30);
 951         zoc3 = Z_RSHIFT((-0x0cef1502LL * zoo1) + (0x01207a8eLL * zoo2) +
 952             (0x01e936dbLL * zoo3), 30);
 953         zoc4 = Z_RSHIFT((0x029fe643LL * zoe1) + (-0x03ef3fc8LL * zoe2) +
 954             (0x014f5923LL * zoe3), 30);
 955         zoc5 = Z_RSHIFT((0x043a9d08LL * zoo1) + (-0x02154febLL * zoo2) +
 956             (0x00670dbdLL * zoo3), 30);
 957
 958         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 959             (int64_t)zoc5 * zoz, Z_SHIFT) +
 960             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 961             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 962 #elif defined(Z_COEFF_INTERP_OPT2X)
 963         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
 964         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
 965
 966         /* 6-point, 5th-order Optimal 2x */
 967         zoz = z - (Z_ONE >> 1);
 968         zoe1 = z_coeff[1] + z_coeff[0];
 969         zoe2 = z_coeff[2] + z_coeff[-1];
 970         zoe3 = z_coeff[3] + z_coeff[-2];
 971         zoo1 = z_coeff[1] - z_coeff[0];
 972         zoo2 = z_coeff[2] - z_coeff[-1];
 973         zoo3 = z_coeff[3] - z_coeff[-2];
 974
 975         zoc0 = Z_RSHIFT((0x19edb6fdLL * zoe1) + (0x05ebd062LL * zoe2) +
 976             (0x00267881LL * zoe3), 30);
 977         zoc1 = Z_RSHIFT((0x1223af76LL * zoo1) + (0x0de3dd6bLL * zoo2) +
 978             (0x00d683cdLL * zoo3), 30);
 979         zoc2 = Z_RSHIFT((-0x0c3ee068LL * zoe1) + (0x0a5c3769LL * zoe2) +
 980             (0x01e2aceaLL * zoe3), 30);
 981         zoc3 = Z_RSHIFT((-0x0a8ab614LL * zoo1) + (-0x0019522eLL * zoo2) +
 982             (0x022cefc7LL * zoo3), 30);
 983         zoc4 = Z_RSHIFT((0x0276187dLL * zoe1) + (-0x03a801e8LL * zoe2) +
 984             (0x0131d935LL * zoe3), 30);
 985         zoc5 = Z_RSHIFT((0x02c373f5LL * zoo1) + (-0x01275f83LL * zoo2) +
 986             (0x0018ee79LL * zoo3), 30);
 987
 988         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
 989             (int64_t)zoc5 * zoz, Z_SHIFT) +
 990             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
 991             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
 992 #else
 993 #error "Interpolation type screwed!"
 994 #endif
 995
 996 #if Z_POLYPHASE_COEFF_SHIFT > 0
 997         coeff = Z_RSHIFT(coeff, Z_POLYPHASE_COEFF_SHIFT);
 998 #endif
 999         return (coeff);
1000 }
1001
1002 static int
1003 z_resampler_build_polyphase(struct z_info *info)
1004 {
1005         int32_t alpha, c, i, z, idx;
1006
1007         /* Let this be here first. */
1008         if (info->z_pcoeff != NULL) {
1009                 kfree(info->z_pcoeff, M_DEVBUF);
1010                 info->z_pcoeff = NULL;
1011         }
1012
1013         if (feeder_rate_polyphase_max < 1)
1014                 return (ENOTSUP);
1015
1016         if (((int64_t)info->z_size * info->z_gy * 2) >
1017             feeder_rate_polyphase_max) {
1018 #ifndef _KERNEL
1019                 fprintf(stderr, "Polyphase entries exceed: [%d/%d] %jd > %d\n",
1020                     info->z_gx, info->z_gy,
1021                     (intmax_t)info->z_size * info->z_gy * 2,
1022                     feeder_rate_polyphase_max);
1023 #endif
1024                 return (E2BIG);
1025         }
1026
1027         info->z_pcoeff = kmalloc(sizeof(int32_t) *
1028             info->z_size * info->z_gy * 2, M_DEVBUF, M_WAITOK | M_ZERO);
1029         if (info->z_pcoeff == NULL)
1030                 return (ENOMEM);
1031
1032         for (alpha = 0; alpha < info->z_gy; alpha++) {
1033                 z = alpha * info->z_dx;
1034                 c = 0;
1035                 for (i = info->z_size; i != 0; i--) {
1036                         c += z >> Z_SHIFT;
1037                         z &= Z_MASK;
1038                         idx = (alpha * info->z_size * 2) +
1039                             (info->z_size * 2) - i;
1040                         info->z_pcoeff[idx] =
1041                             z_coeff_interpolate(z, info->z_coeff + c);
1042                         z += info->z_dy;
1043                 }
1044                 z = info->z_dy - (alpha * info->z_dx);
1045                 c = 0;
1046                 for (i = info->z_size; i != 0; i--) {
1047                         c += z >> Z_SHIFT;
1048                         z &= Z_MASK;
1049                         idx = (alpha * info->z_size * 2) + i - 1;
1050                         info->z_pcoeff[idx] =
1051                             z_coeff_interpolate(z, info->z_coeff + c);
1052                         z += info->z_dy;
1053                 }
1054         }
1055
1056 #ifndef _KERNEL
1057         fprintf(stderr, "Polyphase: [%d/%d] %d entries\n",
1058             info->z_gx, info->z_gy, info->z_size * info->z_gy * 2);
1059 #endif
1060
1061         return (0);
1062 }
1063
1064 static int
1065 z_resampler_setup(struct pcm_feeder *f)
1066 {
1067         struct z_info *info;
1068         int64_t gy2gx_max, gx2gy_max;
1069         uint32_t format;
1070         int32_t align, i, z_scale;
1071         int adaptive;
1072
1073         info = f->data;
1074         z_resampler_reset(info);
1075
1076         if (info->src == info->dst)
1077                 return (0);
1078
1079         /* Shrink by greatest common divisor. */
1080         i = z_gcd(info->src, info->dst);
1081         info->z_gx = info->src / i;
1082         info->z_gy = info->dst / i;
1083
1084         /* Too big, or too small. Bail out. */
1085         if (!(Z_FACTOR_SAFE(info->z_gx) && Z_FACTOR_SAFE(info->z_gy)))
1086                 return (EINVAL);
1087
1088         format = f->desc->in;
1089         adaptive = 0;
1090         z_scale = 0;
1091
1092         /*
1093          * Setup everything: filter length, conversion factor, etc.
1094          */
1095         if (Z_IS_SINC(info)) {
1096                 /*
1097                  * Downsampling, or upsampling scaling factor. As long as the
1098                  * factor can be represented by a fraction of 1 << Z_SHIFT,
1099                  * we're pretty much in business. Scaling is not needed for
1100                  * upsampling, so we just slap Z_ONE there.
1101                  */
1102                 if (info->z_gx > info->z_gy)
1103                         /*
1104                          * If the downsampling ratio is beyond sanity,
1105                          * enable semi-adaptive mode. Although handling
1106                          * extreme ratio is possible, the result of the
1107                          * conversion is just pointless, unworthy,
1108                          * nonsensical noises, etc.
1109                          */
1110                         if ((info->z_gx / info->z_gy) > Z_SINC_DOWNMAX)
1111                                 z_scale = Z_ONE / Z_SINC_DOWNMAX;
1112                         else
1113                                 z_scale = ((uint64_t)info->z_gy << Z_SHIFT) /
1114                                     info->z_gx;
1115                 else
1116                         z_scale = Z_ONE;
1117
1118                 /*
1119                  * This is actually impossible, unless anything above
1120                  * overflow.
1121                  */
1122                 if (z_scale < 1)
1123                         return (E2BIG);
1124
1125                 /*
1126                  * Calculate sample time/coefficients index drift. It is
1127                  * a constant for upsampling, but downsampling require
1128                  * heavy duty filtering with possible too long filters.
1129                  * If anything goes wrong, revisit again and enable
1130                  * adaptive mode.
1131                  */
1132 z_setup_adaptive_sinc:
1133                 if (info->z_pcoeff != NULL) {
1134                         kfree(info->z_pcoeff, M_DEVBUF);
1135                         info->z_pcoeff = NULL;
1136                 }
1137
1138                 if (adaptive == 0) {
1139                         info->z_dy = z_scale << Z_DRIFT_SHIFT;
1140                         if (info->z_dy < 1)
1141                                 return (E2BIG);
1142                         info->z_scale = z_scale;
1143                 } else {
1144                         info->z_dy = Z_FULL_ONE;
1145                         info->z_scale = Z_ONE;
1146                 }
1147
1148 #if 0
1149 #define Z_SCALE_DIV     10000
1150 #define Z_SCALE_LIMIT(s, v)                                             \
1151         ((((uint64_t)(s) * (v)) + (Z_SCALE_DIV >> 1)) / Z_SCALE_DIV)
1152
1153                 info->z_scale = Z_SCALE_LIMIT(info->z_scale, 9780);
1154 #endif
1155
1156                 /* Smallest drift increment. */
1157                 info->z_dx = info->z_dy / info->z_gy;
1158
1159                 /*
1160                  * Overflow or underflow. Try adaptive, let it continue and
1161                  * retry.
1162                  */
1163                 if (info->z_dx < 1) {
1164                         if (adaptive == 0) {
1165                                 adaptive = 1;
1166                                 goto z_setup_adaptive_sinc;
1167                         }
1168                         return (E2BIG);
1169                 }
1170
1171                 /*
1172                  * Round back output drift.
1173                  */
1174                 info->z_dy = info->z_dx * info->z_gy;
1175
1176                 for (i = 0; i < Z_COEFF_TAB_SIZE; i++) {
1177                         if (Z_SINC_COEFF_IDX(info) != i)
1178                                 continue;
1179                         /*
1180                          * Calculate required filter length and guard
1181                          * against possible abusive result. Note that
1182                          * this represents only 1/2 of the entire filter
1183                          * length.
1184                          */
1185                         info->z_size = z_resampler_sinc_len(info);
1186
1187                         /*
1188                          * Multiple of 2 rounding, for better accumulator
1189                          * performance.
1190                          */
1191                         info->z_size &= ~1;
1192
1193                         if (info->z_size < 2 || info->z_size > Z_SINC_MAX) {
1194                                 if (adaptive == 0) {
1195                                         adaptive = 1;
1196                                         goto z_setup_adaptive_sinc;
1197                                 }
1198                                 return (E2BIG);
1199                         }
1200                         info->z_coeff = z_coeff_tab[i].coeff + Z_COEFF_OFFSET;
1201                         info->z_dcoeff = z_coeff_tab[i].dcoeff;
1202                         break;
1203                 }
1204
1205                 if (info->z_coeff == NULL || info->z_dcoeff == NULL)
1206                         return (EINVAL);
1207         } else if (Z_IS_LINEAR(info)) {
1208                 /*
1209                  * Don't put much effort if we're doing linear interpolation.
1210                  * Just center the interpolation distance within Z_LINEAR_ONE,
1211                  * and be happy about it.
1212                  */
1213                 info->z_dx = Z_LINEAR_FULL_ONE / info->z_gy;
1214         }
1215
1216         /*
1217          * We're safe for now, lets continue.. Look for our resampler
1218          * depending on configured format and quality.
1219          */
1220         for (i = 0; i < Z_RESAMPLER_TAB_SIZE; i++) {
1221                 int ridx;
1222
1223                 if (AFMT_ENCODING(format) != z_resampler_tab[i].format)
1224                         continue;
1225                 if (Z_IS_SINC(info) && adaptive == 0 &&
1226                     z_resampler_build_polyphase(info) == 0)
1227                         ridx = Z_RESAMPLER_SINC_POLYPHASE;
1228                 else
1229                         ridx = Z_RESAMPLER_IDX(info);
1230                 info->z_resample = z_resampler_tab[i].resampler[ridx];
1231                 break;
1232         }
1233
1234         if (info->z_resample == NULL)
1235                 return (EINVAL);
1236
1237         info->bps = AFMT_BPS(format);
1238         align = info->channels * info->bps;
1239
1240         /*
1241          * Calculate largest value that can be fed into z_gy2gx() and
1242          * z_gx2gy() without causing (signed) 32bit overflow. z_gy2gx() will
1243          * be called early during feeding process to determine how much input
1244          * samples that is required to generate requested output, while
1245          * z_gx2gy() will be called just before samples filtering /
1246          * accumulation process based on available samples that has been
1247          * calculated using z_gx2gy().
1248          *
1249          * Now that is damn confusing, I guess ;-) .
1250          */
1251         gy2gx_max = (((uint64_t)info->z_gy * INT32_MAX) - info->z_gy + 1) /
1252             info->z_gx;
1253
1254         if ((gy2gx_max * align) > SND_FXDIV_MAX)
1255                 gy2gx_max = SND_FXDIV_MAX / align;
1256
1257         if (gy2gx_max < 1)
1258                 return (E2BIG);
1259
1260         gx2gy_max = (((uint64_t)info->z_gx * INT32_MAX) - info->z_gy) /
1261             info->z_gy;
1262
1263         if (gx2gy_max > INT32_MAX)
1264                 gx2gy_max = INT32_MAX;
1265
1266         if (gx2gy_max < 1)
1267                 return (E2BIG);
1268
1269         /*
1270          * Ensure that z_gy2gx() at its largest possible calculated value
1271          * (alpha = 0) will not cause overflow further late during z_gx2gy()
1272          * stage.
1273          */
1274         if (z_gy2gx(info, gy2gx_max) > _Z_GCAST(gx2gy_max))
1275                 return (E2BIG);
1276
1277         info->z_maxfeed = gy2gx_max * align;
1278
1279 #ifdef Z_USE_ALPHADRIFT
1280         info->z_startdrift = z_gy2gx(info, 1);
1281         info->z_alphadrift = z_drift(info, info->z_startdrift, 1);
1282 #endif
1283
1284         i = z_gy2gx(info, 1);
1285         info->z_full = z_roundpow2((info->z_size << 1) + i);
1286
1287         /*
1288          * Too big to be true, and overflowing left and right like mad ..
1289          */
1290         if ((info->z_full * align) < 1) {
1291                 if (adaptive == 0 && Z_IS_SINC(info)) {
1292                         adaptive = 1;
1293                         goto z_setup_adaptive_sinc;
1294                 }
1295                 return (E2BIG);
1296         }
1297
1298         /*
1299          * Increase full buffer size if its too small to reduce cyclic
1300          * buffer shifting in main conversion/feeder loop.
1301          */
1302         while (info->z_full < Z_RESERVOIR_MAX &&
1303             (info->z_full - (info->z_size << 1)) < Z_RESERVOIR)
1304                 info->z_full <<= 1;
1305
1306         /* Initialize buffer position. */
1307         info->z_mask = info->z_full - 1;
1308         info->z_start = z_prev(info, info->z_size << 1, 1);
1309         info->z_pos = z_next(info, info->z_start, 1);
1310
1311         /*
1312          * Allocate or reuse delay line buffer, whichever makes sense.
1313          */
1314         i = info->z_full * align;
1315         if (i < 1)
1316                 return (E2BIG);
1317
1318         if (info->z_delay == NULL || info->z_alloc < i ||
1319             i <= (info->z_alloc >> 1)) {
1320                 if (info->z_delay != NULL)
1321                         kfree(info->z_delay, M_DEVBUF);
1322                 info->z_delay = kmalloc(i, M_DEVBUF, M_WAITOK | M_ZERO);
1323                 if (info->z_delay == NULL)
1324                         return (ENOMEM);
1325                 info->z_alloc = i;
1326         }
1327
1328         /*
1329          * Zero out head of buffer to avoid pops and clicks.
1330          */
1331         memset(info->z_delay, sndbuf_zerodata(f->desc->out),
1332             info->z_pos * align);
1333
1334 #ifdef Z_DIAGNOSTIC
1335         /*
1336          * XXX Debuging mess !@#$%^
1337          */
1338 #define dumpz(x)        fprintf(stderr, "\t%12s = %10u : %-11d\n",      \
1339                             "z_"__STRING(x), (uint32_t)info->z_##x,     \
1340                             (int32_t)info->z_##x)
1341         fprintf(stderr, "\n%s():\n", __func__);
1342         fprintf(stderr, "\tchannels=%d, bps=%d, format=0x%08x, quality=%d\n",
1343             info->channels, info->bps, format, info->quality);
1344         fprintf(stderr, "\t%d (%d) -> %d (%d), ",
1345             info->src, info->rsrc, info->dst, info->rdst);
1346         fprintf(stderr, "[%d/%d]\n", info->z_gx, info->z_gy);
1347         fprintf(stderr, "\tminreq=%d, ", z_gy2gx(info, 1));
1348         if (adaptive != 0)
1349                 z_scale = Z_ONE;
1350         fprintf(stderr, "factor=0x%08x/0x%08x (%f)\n",
1351             z_scale, Z_ONE, (double)z_scale / Z_ONE);
1352         fprintf(stderr, "\tbase_length=%d, ", Z_SINC_BASE_LEN(info));
1353         fprintf(stderr, "adaptive=%s\n", (adaptive != 0) ? "YES" : "NO");
1354         dumpz(size);
1355         dumpz(alloc);
1356         if (info->z_alloc < 1024)
1357                 fprintf(stderr, "\t%15s%10d Bytes\n",
1358                     "", info->z_alloc);
1359         else if (info->z_alloc < (1024 << 10))
1360                 fprintf(stderr, "\t%15s%10d KBytes\n",
1361                     "", info->z_alloc >> 10);
1362         else if (info->z_alloc < (1024 << 20))
1363                 fprintf(stderr, "\t%15s%10d MBytes\n",
1364                     "", info->z_alloc >> 20);
1365         else
1366                 fprintf(stderr, "\t%15s%10d GBytes\n",
1367                     "", info->z_alloc >> 30);
1368         fprintf(stderr, "\t%12s   %10d (min output samples)\n",
1369             "",
1370             (int32_t)z_gx2gy(info, info->z_full - (info->z_size << 1)));
1371         fprintf(stderr, "\t%12s   %10d (min allocated output samples)\n",
1372             "",
1373             (int32_t)z_gx2gy(info, (info->z_alloc / align) -
1374             (info->z_size << 1)));
1375         fprintf(stderr, "\t%12s = %10d\n",
1376             "z_gy2gx()", (int32_t)z_gy2gx(info, 1));
1377         fprintf(stderr, "\t%12s = %10d -> z_gy2gx() -> %d\n",
1378             "Max", (int32_t)gy2gx_max, (int32_t)z_gy2gx(info, gy2gx_max));
1379         fprintf(stderr, "\t%12s = %10d\n",
1380             "z_gx2gy()", (int32_t)z_gx2gy(info, 1));
1381         fprintf(stderr, "\t%12s = %10d -> z_gx2gy() -> %d\n",
1382             "Max", (int32_t)gx2gy_max, (int32_t)z_gx2gy(info, gx2gy_max));
1383         dumpz(maxfeed);
1384         dumpz(full);
1385         dumpz(start);
1386         dumpz(pos);
1387         dumpz(scale);
1388         fprintf(stderr, "\t%12s   %10f\n", "",
1389             (double)info->z_scale / Z_ONE);
1390         dumpz(dx);
1391         fprintf(stderr, "\t%12s   %10f\n", "",
1392             (double)info->z_dx / info->z_dy);
1393         dumpz(dy);
1394         fprintf(stderr, "\t%12s   %10d (drift step)\n", "",
1395             info->z_dy >> Z_SHIFT);
1396         fprintf(stderr, "\t%12s   %10d (scaling differences)\n", "",
1397             (z_scale << Z_DRIFT_SHIFT) - info->z_dy);
1398         fprintf(stderr, "\t%12s = %u bytes\n",
1399             "intpcm32_t", sizeof(intpcm32_t));
1400         fprintf(stderr, "\t%12s = 0x%08x, smallest=%.16lf\n",
1401             "Z_ONE", Z_ONE, (double)1.0 / (double)Z_ONE);
1402 #endif
1403
1404         return (0);
1405 }
1406
1407 static int
1408 z_resampler_set(struct pcm_feeder *f, int what, int32_t value)
1409 {
1410         struct z_info *info;
1411         int32_t oquality;
1412
1413         info = f->data;
1414
1415         switch (what) {
1416         case Z_RATE_SRC:
1417                 if (value < feeder_rate_min || value > feeder_rate_max)
1418                         return (E2BIG);
1419                 if (value == info->rsrc)
1420                         return (0);
1421                 info->rsrc = value;
1422                 break;
1423         case Z_RATE_DST:
1424                 if (value < feeder_rate_min || value > feeder_rate_max)
1425                         return (E2BIG);
1426                 if (value == info->rdst)
1427                         return (0);
1428                 info->rdst = value;
1429                 break;
1430         case Z_RATE_QUALITY:
1431                 if (value < Z_QUALITY_MIN || value > Z_QUALITY_MAX)
1432                         return (EINVAL);
1433                 if (value == info->quality)
1434                         return (0);
1435                 /*
1436                  * If we failed to set the requested quality, restore
1437                  * the old one. We cannot afford leaving it broken since
1438                  * passive feeder chains like vchans never reinitialize
1439                  * itself.
1440                  */
1441                 oquality = info->quality;
1442                 info->quality = value;
1443                 if (z_resampler_setup(f) == 0)
1444                         return (0);
1445                 info->quality = oquality;
1446                 break;
1447         case Z_RATE_CHANNELS:
1448                 if (value < SND_CHN_MIN || value > SND_CHN_MAX)
1449                         return (EINVAL);
1450                 if (value == info->channels)
1451                         return (0);
1452                 info->channels = value;
1453                 break;
1454         default:
1455                 return (EINVAL);
1456                 break;
1457         }
1458
1459         return (z_resampler_setup(f));
1460 }
1461
1462 static int
1463 z_resampler_get(struct pcm_feeder *f, int what)
1464 {
1465         struct z_info *info;
1466
1467         info = f->data;
1468
1469         switch (what) {
1470         case Z_RATE_SRC:
1471                 return (info->rsrc);
1472                 break;
1473         case Z_RATE_DST:
1474                 return (info->rdst);
1475                 break;
1476         case Z_RATE_QUALITY:
1477                 return (info->quality);
1478                 break;
1479         case Z_RATE_CHANNELS:
1480                 return (info->channels);
1481                 break;
1482         default:
1483                 break;
1484         }
1485
1486         return (-1);
1487 }
1488
1489 static int
1490 z_resampler_init(struct pcm_feeder *f)
1491 {
1492         struct z_info *info;
1493         int ret;
1494
1495         if (f->desc->in != f->desc->out)
1496                 return (EINVAL);
1497
1498         info = kmalloc(sizeof(*info), M_DEVBUF, M_WAITOK | M_ZERO);
1499         if (info == NULL)
1500                 return (ENOMEM);
1501
1502         info->rsrc = Z_RATE_DEFAULT;
1503         info->rdst = Z_RATE_DEFAULT;
1504         info->quality = feeder_rate_quality;
1505         info->channels = AFMT_CHANNEL(f->desc->in);
1506
1507         f->data = info;
1508
1509         ret = z_resampler_setup(f);
1510         if (ret != 0) {
1511                 if (info->z_pcoeff != NULL)
1512                         kfree(info->z_pcoeff, M_DEVBUF);
1513                 if (info->z_delay != NULL)
1514                         kfree(info->z_delay, M_DEVBUF);
1515                 kfree(info, M_DEVBUF);
1516                 f->data = NULL;
1517         }
1518
1519         return (ret);
1520 }
1521
1522 static int
1523 z_resampler_free(struct pcm_feeder *f)
1524 {
1525         struct z_info *info;
1526
1527         info = f->data;
1528         if (info != NULL) {
1529                 if (info->z_pcoeff != NULL)
1530                         kfree(info->z_pcoeff, M_DEVBUF);
1531                 if (info->z_delay != NULL)
1532                         kfree(info->z_delay, M_DEVBUF);
1533                 kfree(info, M_DEVBUF);
1534         }
1535
1536         f->data = NULL;
1537
1538         return (0);
1539 }
1540
1541 static uint32_t
1542 z_resampler_feed_internal(struct pcm_feeder *f, struct pcm_channel *c,
1543     uint8_t *b, uint32_t count, void *source)
1544 {
1545         struct z_info *info;
1546         int32_t alphadrift, startdrift, reqout, ocount, reqin, align;
1547         int32_t fetch, fetched, start, cp;
1548         uint8_t *dst;
1549
1550         info = f->data;
1551         if (info->z_resample == NULL)
1552                 return (z_feed(f->source, c, b, count, source));
1553
1554         /*
1555          * Calculate sample size alignment and amount of sample output.
1556          * We will do everything in sample domain, but at the end we
1557          * will jump back to byte domain.
1558          */
1559         align = info->channels * info->bps;
1560         ocount = SND_FXDIV(count, align);
1561         if (ocount == 0)
1562                 return (0);
1563
1564         /*
1565          * Calculate amount of input samples that is needed to generate
1566          * exact amount of output.
1567          */
1568         reqin = z_gy2gx(info, ocount) - z_fetched(info);
1569
1570 #ifdef Z_USE_ALPHADRIFT
1571         startdrift = info->z_startdrift;
1572         alphadrift = info->z_alphadrift;
1573 #else
1574         startdrift = _Z_GY2GX(info, 0, 1);
1575         alphadrift = z_drift(info, startdrift, 1);
1576 #endif
1577
1578         dst = b;
1579
1580         do {
1581                 if (reqin != 0) {
1582                         fetch = z_min(z_free(info), reqin);
1583                         if (fetch == 0) {
1584                                 /*
1585                                  * No more free spaces, so wind enough
1586                                  * samples back to the head of delay line
1587                                  * in byte domain.
1588                                  */
1589                                 fetched = z_fetched(info);
1590                                 start = z_prev(info, info->z_start,
1591                                     (info->z_size << 1) - 1);
1592                                 cp = (info->z_size << 1) + fetched;
1593                                 z_copy(info->z_delay + (start * align),
1594                                     info->z_delay, cp * align);
1595                                 info->z_start =
1596                                     z_prev(info, info->z_size << 1, 1);
1597                                 info->z_pos =
1598                                     z_next(info, info->z_start, fetched + 1);
1599                                 fetch = z_min(z_free(info), reqin);
1600 #ifdef Z_DIAGNOSTIC
1601                                 if (1) {
1602                                         static uint32_t kk = 0;
1603                                         fprintf(stderr,
1604                                             "Buffer Move: "
1605                                             "start=%d fetched=%d cp=%d "
1606                                             "cycle=%u [%u]\r",
1607                                             start, fetched, cp, info->z_cycle,
1608                                             ++kk);
1609                                 }
1610                                 info->z_cycle = 0;
1611 #endif
1612                         }
1613                         if (fetch != 0) {
1614                                 /*
1615                                  * Fetch in byte domain and jump back
1616                                  * to sample domain.
1617                                  */
1618                                 fetched = SND_FXDIV(z_feed(f->source, c,
1619                                     info->z_delay + (info->z_pos * align),
1620                                     fetch * align, source), align);
1621                                 /*
1622                                  * Prepare to convert fetched buffer,
1623                                  * or mark us done if we cannot fulfill
1624                                  * the request.
1625                                  */
1626                                 reqin -= fetched;
1627                                 info->z_pos += fetched;
1628                                 if (fetched != fetch)
1629                                         reqin = 0;
1630                         }
1631                 }
1632
1633                 reqout = z_min(z_gx2gy(info, z_fetched(info)), ocount);
1634                 if (reqout != 0) {
1635                         ocount -= reqout;
1636
1637                         /*
1638                          * Drift.. drift.. drift..
1639                          *
1640                          * Notice that there are 2 methods of doing the drift
1641                          * operations: The former is much cleaner (in a sense
1642                          * of mathematical readings of my eyes), but slower
1643                          * due to integer division in z_gy2gx(). Nevertheless,
1644                          * both should give the same exact accurate drifting
1645                          * results, so the later is favourable.
1646                          */
1647                         do {
1648                                 info->z_resample(info, dst);
1649 #if 0
1650                                 startdrift = z_gy2gx(info, 1);
1651                                 alphadrift = z_drift(info, startdrift, 1);
1652                                 info->z_start += startdrift;
1653                                 info->z_alpha += alphadrift;
1654 #else
1655                                 info->z_alpha += alphadrift;
1656                                 if (info->z_alpha < info->z_gy)
1657                                         info->z_start += startdrift;
1658                                 else {
1659                                         info->z_start += startdrift - 1;
1660                                         info->z_alpha -= info->z_gy;
1661                                 }
1662 #endif
1663                                 dst += align;
1664 #ifdef Z_DIAGNOSTIC
1665                                 info->z_cycle++;
1666 #endif
1667                         } while (--reqout != 0);
1668                 }
1669         } while (reqin != 0 && ocount != 0);
1670
1671         /*
1672          * Back to byte domain..
1673          */
1674         return (dst - b);
1675 }
1676
1677 static int
1678 z_resampler_feed(struct pcm_feeder *f, struct pcm_channel *c, uint8_t *b,
1679     uint32_t count, void *source)
1680 {
1681         uint32_t feed, maxfeed, left;
1682
1683         /*
1684          * Split count to smaller chunks to avoid possible 32bit overflow.
1685          */
1686         maxfeed = ((struct z_info *)(f->data))->z_maxfeed;
1687         left = count;
1688
1689         do {
1690                 feed = z_resampler_feed_internal(f, c, b,
1691                     z_min(maxfeed, left), source);
1692                 b += feed;
1693                 left -= feed;
1694         } while (left != 0 && feed != 0);
1695
1696         return (count - left);
1697 }
1698
1699 static struct pcm_feederdesc feeder_rate_desc[] = {
1700         { FEEDER_RATE, 0, 0, 0, 0 },
1701         { 0, 0, 0, 0, 0 },
1702 };
1703
1704 static kobj_method_t feeder_rate_methods[] = {
1705         KOBJMETHOD(feeder_init,         z_resampler_init),
1706         KOBJMETHOD(feeder_free,         z_resampler_free),
1707         KOBJMETHOD(feeder_set,          z_resampler_set),
1708         KOBJMETHOD(feeder_get,          z_resampler_get),
1709         KOBJMETHOD(feeder_feed,         z_resampler_feed),
1710         KOBJMETHOD_END
1711 };
1712
1713 FEEDER_DECLARE(feeder_rate, NULL);