aba2abfbfdec4a55e5b15b64399481a9bd9524d0
[dragonfly.git] / sys / dev / sound / pcm / feeder_rate.c
1 /*-
2  * Copyright (c) 2005-2009 Ariff Abdullah <ariff@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26
27 /*
28  * feeder_rate: (Codename: Z Resampler), which means any effort to create
29  *              future replacement for this resampler are simply absurd unless
30  *              the world decide to add new alphabet after Z.
31  *
32  * FreeBSD bandlimited sinc interpolator, technically based on
33  * "Digital Audio Resampling" by Julius O. Smith III
34  *  - http://ccrma.stanford.edu/~jos/resample/
35  *
36  * The Good:
37  * + all out fixed point integer operations, no soft-float or anything like
38  *   that.
39  * + classic polyphase converters with high quality coefficient's polynomial
40  *   interpolators.
41  * + fast, faster, or the fastest of its kind.
42  * + compile time configurable.
43  * + etc etc..
44  *
45  * The Bad:
46  * - The z, z_, and Z_ . Due to mental block (or maybe just 0x7a69), I
47  *   couldn't think of anything simpler than that (feeder_rate_xxx is just
48  *   too long). Expect possible clashes with other zitizens (any?).
49  */
50
51 #ifdef _KERNEL
52 #ifdef HAVE_KERNEL_OPTION_HEADERS
53 #include "opt_snd.h"
54 #endif
55 #include <dev/sound/pcm/sound.h>
56 #include <dev/sound/pcm/pcm.h>
57 #include "feeder_if.h"
58
59 #define SND_USE_FXDIV
60 #include "snd_fxdiv_gen.h"
61
62 SND_DECLARE_FILE("$FreeBSD: head/sys/dev/sound/pcm/feeder_rate.c 267992 2014-06-28 03:56:17Z hselasky $");
63 #endif
64
65 #include "feeder_rate_gen.h"
66
67 #if !defined(_KERNEL) && defined(SND_DIAGNOSTIC)
68 #undef Z_DIAGNOSTIC
69 #define Z_DIAGNOSTIC            1
70 #elif defined(_KERNEL)
71 #undef Z_DIAGNOSTIC
72 #endif
73
74 #ifndef Z_QUALITY_DEFAULT
75 #define Z_QUALITY_DEFAULT       Z_QUALITY_LINEAR
76 #endif
77
78 #define Z_RESERVOIR             2048
79 #define Z_RESERVOIR_MAX         131072
80
81 #define Z_SINC_MAX              0x3fffff
82 #define Z_SINC_DOWNMAX          48              /* 384000 / 8000 */
83
84 #ifdef _KERNEL
85 #define Z_POLYPHASE_MAX         183040          /* 286 taps, 640 phases */
86 #else
87 #define Z_POLYPHASE_MAX         1464320         /* 286 taps, 5120 phases */
88 #endif
89
90 #define Z_RATE_DEFAULT          48000
91
92 #define Z_RATE_MIN              FEEDRATE_RATEMIN
93 #define Z_RATE_MAX              FEEDRATE_RATEMAX
94 #define Z_ROUNDHZ               FEEDRATE_ROUNDHZ
95 #define Z_ROUNDHZ_MIN           FEEDRATE_ROUNDHZ_MIN
96 #define Z_ROUNDHZ_MAX           FEEDRATE_ROUNDHZ_MAX
97
98 #define Z_RATE_SRC              FEEDRATE_SRC
99 #define Z_RATE_DST              FEEDRATE_DST
100 #define Z_RATE_QUALITY          FEEDRATE_QUALITY
101 #define Z_RATE_CHANNELS         FEEDRATE_CHANNELS
102
103 #define Z_PARANOID              1
104
105 #define Z_MULTIFORMAT           1
106
107 #ifdef _KERNEL
108 #undef Z_USE_ALPHADRIFT
109 #define Z_USE_ALPHADRIFT        1
110 #endif
111
112 #define Z_FACTOR_MIN            1
113 #define Z_FACTOR_MAX            Z_MASK
114 #define Z_FACTOR_SAFE(v)        (!((v) < Z_FACTOR_MIN || (v) > Z_FACTOR_MAX))
115
116 struct z_info;
117
118 typedef void (*z_resampler_t)(struct z_info *, uint8_t *);
119
120 struct z_info {
121         int32_t rsrc, rdst;     /* original source / destination rates */
122         int32_t src, dst;       /* rounded source / destination rates */
123         int32_t channels;       /* total channels */
124         int32_t bps;            /* bytes-per-sample */
125         int32_t quality;        /* resampling quality */
126
127         int32_t z_gx, z_gy;     /* interpolation / decimation ratio */
128         int32_t z_alpha;        /* output sample time phase / drift */
129         uint8_t *z_delay;       /* FIR delay line / linear buffer */
130         int32_t *z_coeff;       /* FIR coefficients */
131         int32_t *z_dcoeff;      /* FIR coefficients differences */
132         int32_t *z_pcoeff;      /* FIR polyphase coefficients */
133         int32_t z_scale;        /* output scaling */
134         int32_t z_dx;           /* input sample drift increment */
135         int32_t z_dy;           /* output sample drift increment */
136 #ifdef Z_USE_ALPHADRIFT
137         int32_t z_alphadrift;   /* alpha drift rate */
138         int32_t z_startdrift;   /* buffer start position drift rate */
139 #endif
140         int32_t z_mask;         /* delay line full length mask */
141         int32_t z_size;         /* half width of FIR taps */
142         int32_t z_full;         /* full size of delay line */
143         int32_t z_alloc;        /* largest allocated full size of delay line */
144         int32_t z_start;        /* buffer processing start position */
145         int32_t z_pos;          /* current position for the next feed */
146 #ifdef Z_DIAGNOSTIC
147         uint32_t z_cycle;       /* output cycle, purely for statistical */
148 #endif
149         int32_t z_maxfeed;      /* maximum feed to avoid 32bit overflow */
150
151         z_resampler_t z_resample;
152 };
153
154 int feeder_rate_min = Z_RATE_MIN;
155 int feeder_rate_max = Z_RATE_MAX;
156 int feeder_rate_round = Z_ROUNDHZ;
157 int feeder_rate_quality = Z_QUALITY_DEFAULT;
158
159 static int feeder_rate_polyphase_max = Z_POLYPHASE_MAX;
160
161 #ifdef _KERNEL
162 static char feeder_rate_presets[] = FEEDER_RATE_PRESETS;
163 SYSCTL_STRING(_hw_snd, OID_AUTO, feeder_rate_presets, CTLFLAG_RD,
164     &feeder_rate_presets, 0, "compile-time rate presets");
165
166 TUNABLE_INT("hw.snd.feeder_rate_min", &feeder_rate_min);
167 TUNABLE_INT("hw.snd.feeder_rate_max", &feeder_rate_max);
168 TUNABLE_INT("hw.snd.feeder_rate_round", &feeder_rate_round);
169 TUNABLE_INT("hw.snd.feeder_rate_quality", &feeder_rate_quality);
170
171 TUNABLE_INT("hw.snd.feeder_rate_polyphase_max", &feeder_rate_polyphase_max);
172 SYSCTL_INT(_hw_snd, OID_AUTO, feeder_rate_polyphase_max, CTLFLAG_RW,
173     &feeder_rate_polyphase_max, 0, "maximum allowable polyphase entries");
174
175 static int
176 sysctl_hw_snd_feeder_rate_min(SYSCTL_HANDLER_ARGS)
177 {
178         int err, val;
179
180         val = feeder_rate_min;
181         err = sysctl_handle_int(oidp, &val, 0, req);
182
183         if (err != 0 || req->newptr == NULL || val == feeder_rate_min)
184                 return (err);
185
186         if (!(Z_FACTOR_SAFE(val) && val < feeder_rate_max))
187                 return (EINVAL);
188
189         feeder_rate_min = val;
190
191         return (0);
192 }
193 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_min, CTLTYPE_INT | CTLFLAG_RW,
194     0, sizeof(int), sysctl_hw_snd_feeder_rate_min, "I",
195     "minimum allowable rate");
196
197 static int
198 sysctl_hw_snd_feeder_rate_max(SYSCTL_HANDLER_ARGS)
199 {
200         int err, val;
201
202         val = feeder_rate_max;
203         err = sysctl_handle_int(oidp, &val, 0, req);
204
205         if (err != 0 || req->newptr == NULL || val == feeder_rate_max)
206                 return (err);
207
208         if (!(Z_FACTOR_SAFE(val) && val > feeder_rate_min))
209                 return (EINVAL);
210
211         feeder_rate_max = val;
212
213         return (0);
214 }
215 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_max, CTLTYPE_INT | CTLFLAG_RW,
216     0, sizeof(int), sysctl_hw_snd_feeder_rate_max, "I",
217     "maximum allowable rate");
218
219 static int
220 sysctl_hw_snd_feeder_rate_round(SYSCTL_HANDLER_ARGS)
221 {
222         int err, val;
223
224         val = feeder_rate_round;
225         err = sysctl_handle_int(oidp, &val, 0, req);
226
227         if (err != 0 || req->newptr == NULL || val == feeder_rate_round)
228                 return (err);
229
230         if (val < Z_ROUNDHZ_MIN || val > Z_ROUNDHZ_MAX)
231                 return (EINVAL);
232
233         feeder_rate_round = val - (val % Z_ROUNDHZ);
234
235         return (0);
236 }
237 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_round, CTLTYPE_INT | CTLFLAG_RW,
238     0, sizeof(int), sysctl_hw_snd_feeder_rate_round, "I",
239     "sample rate converter rounding threshold");
240
241 static int
242 sysctl_hw_snd_feeder_rate_quality(SYSCTL_HANDLER_ARGS)
243 {
244         struct snddev_info *d;
245         struct pcm_channel *c;
246         struct pcm_feeder *f;
247         int i, err, val;
248
249         val = feeder_rate_quality;
250         err = sysctl_handle_int(oidp, &val, 0, req);
251
252         if (err != 0 || req->newptr == NULL || val == feeder_rate_quality)
253                 return (err);
254
255         if (val < Z_QUALITY_MIN || val > Z_QUALITY_MAX)
256                 return (EINVAL);
257
258         feeder_rate_quality = val;
259
260         /*
261          * Traverse all available channels on each device and try to
262          * set resampler quality if and only if it is exist as
263          * part of feeder chains and the channel is idle.
264          */
265         for (i = 0; pcm_devclass != NULL &&
266             i < devclass_get_maxunit(pcm_devclass); i++) {
267                 d = devclass_get_softc(pcm_devclass, i);
268                 if (!PCM_REGISTERED(d))
269                         continue;
270                 PCM_LOCK(d);
271                 PCM_WAIT(d);
272                 PCM_ACQUIRE(d);
273                 CHN_FOREACH(c, d, channels.pcm) {
274                         CHN_LOCK(c);
275                         f = chn_findfeeder(c, FEEDER_RATE);
276                         if (f == NULL || f->data == NULL || CHN_STARTED(c)) {
277                                 CHN_UNLOCK(c);
278                                 continue;
279                         }
280                         (void)FEEDER_SET(f, FEEDRATE_QUALITY, val);
281                         CHN_UNLOCK(c);
282                 }
283                 PCM_RELEASE(d);
284                 PCM_UNLOCK(d);
285         }
286
287         return (0);
288 }
289 SYSCTL_PROC(_hw_snd, OID_AUTO, feeder_rate_quality, CTLTYPE_INT | CTLFLAG_RW,
290     0, sizeof(int), sysctl_hw_snd_feeder_rate_quality, "I",
291     "sample rate converter quality ("__XSTRING(Z_QUALITY_MIN)"=low .. "
292     __XSTRING(Z_QUALITY_MAX)"=high)");
293 #endif  /* _KERNEL */
294
295
296 /*
297  * Resampler type.
298  */
299 #define Z_IS_ZOH(i)             ((i)->quality == Z_QUALITY_ZOH)
300 #define Z_IS_LINEAR(i)          ((i)->quality == Z_QUALITY_LINEAR)
301 #define Z_IS_SINC(i)            ((i)->quality > Z_QUALITY_LINEAR)
302
303 /*
304  * Macroses for accurate sample time drift calculations.
305  *
306  * gy2gx : given the amount of output, return the _exact_ required amount of
307  *         input.
308  * gx2gy : given the amount of input, return the _maximum_ amount of output
309  *         that will be generated.
310  * drift : given the amount of input and output, return the elapsed
311  *         sample-time.
312  */
313 #define _Z_GCAST(x)             ((uint64_t)(x))
314
315 #if defined(__GNUCLIKE_ASM) && defined(__i386__)
316 /*
317  * This is where i386 being beaten to a pulp. Fortunately this function is
318  * rarely being called and if it is, it will decide the best (hopefully)
319  * fastest way to do the division. If we can ensure that everything is dword
320  * aligned, letting the compiler to call udivdi3 to do the division can be
321  * faster compared to this.
322  *
323  * amd64 is the clear winner here, no question about it.
324  */
325 static __inline uint32_t
326 Z_DIV(uint64_t v, uint32_t d)
327 {
328         uint32_t hi, lo, quo, rem;
329
330         hi = v >> 32;
331         lo = v & 0xffffffff;
332
333         /*
334          * As much as we can, try to avoid long division like a plague.
335          */
336         if (hi == 0)
337                 quo = lo / d;
338         else
339                 __asm("divl %2"
340                     : "=a" (quo), "=d" (rem)
341                     : "r" (d), "0" (lo), "1" (hi));
342
343         return (quo);
344 }
345 #else
346 #define Z_DIV(x, y)             ((x) / (y))
347 #endif
348
349 #define _Z_GY2GX(i, a, v)                                               \
350         Z_DIV(((_Z_GCAST((i)->z_gx) * (v)) + ((i)->z_gy - (a) - 1)),    \
351         (i)->z_gy)
352
353 #define _Z_GX2GY(i, a, v)                                               \
354         Z_DIV(((_Z_GCAST((i)->z_gy) * (v)) + (a)), (i)->z_gx)
355
356 #define _Z_DRIFT(i, x, y)                                               \
357         ((_Z_GCAST((i)->z_gy) * (x)) - (_Z_GCAST((i)->z_gx) * (y)))
358
359 #define z_gy2gx(i, v)           _Z_GY2GX(i, (i)->z_alpha, v)
360 #define z_gx2gy(i, v)           _Z_GX2GY(i, (i)->z_alpha, v)
361 #define z_drift(i, x, y)        _Z_DRIFT(i, x, y)
362
363 /*
364  * Macroses for SINC coefficients table manipulations.. whatever.
365  */
366 #define Z_SINC_COEFF_IDX(i)     ((i)->quality - Z_QUALITY_LINEAR - 1)
367
368 #define Z_SINC_LEN(i)                                                   \
369         ((int32_t)(((uint64_t)z_coeff_tab[Z_SINC_COEFF_IDX(i)].len <<   \
370             Z_SHIFT) / (i)->z_dy))
371
372 #define Z_SINC_BASE_LEN(i)                                              \
373         ((z_coeff_tab[Z_SINC_COEFF_IDX(i)].len - 1) >> (Z_DRIFT_SHIFT - 1))
374
375 /*
376  * Macroses for linear delay buffer operations. Alignment is not
377  * really necessary since we're not using true circular buffer, but it
378  * will help us guard against possible trespasser. To be honest,
379  * the linear block operations does not need guarding at all due to
380  * accurate drifting!
381  */
382 #define z_align(i, v)           ((v) & (i)->z_mask)
383 #define z_next(i, o, v)         z_align(i, (o) + (v))
384 #define z_prev(i, o, v)         z_align(i, (o) - (v))
385 #define z_fetched(i)            (z_align(i, (i)->z_pos - (i)->z_start) - 1)
386 #define z_free(i)               ((i)->z_full - (i)->z_pos)
387
388 /*
389  * Macroses for Bla Bla .. :)
390  */
391 #define z_copy(src, dst, sz)    (void)memcpy(dst, src, sz)
392 #define z_feed(...)             FEEDER_FEED(__VA_ARGS__)
393
394 static __inline uint32_t
395 z_min(uint32_t x, uint32_t y)
396 {
397
398         return ((x < y) ? x : y);
399 }
400
401 static int32_t
402 z_gcd(int32_t x, int32_t y)
403 {
404         int32_t w;
405
406         while (y != 0) {
407                 w = x % y;
408                 x = y;
409                 y = w;
410         }
411
412         return (x);
413 }
414
415 static int32_t
416 z_roundpow2(int32_t v)
417 {
418         int32_t i;
419
420         i = 1;
421
422         /*
423          * Let it overflow at will..
424          */
425         while (i > 0 && i < v)
426                 i <<= 1;
427
428         return (i);
429 }
430
431 /*
432  * Zero Order Hold, the worst of the worst, an insult against quality,
433  * but super fast.
434  */
435 static void
436 z_feed_zoh(struct z_info *info, uint8_t *dst)
437 {
438 #if 0
439         z_copy(info->z_delay +
440             (info->z_start * info->channels * info->bps), dst,
441             info->channels * info->bps);
442 #else
443         uint32_t cnt;
444         uint8_t *src;
445
446         cnt = info->channels * info->bps;
447         src = info->z_delay + (info->z_start * cnt);
448
449         /*
450          * This is a bit faster than doing bcopy() since we're dealing
451          * with possible unaligned samples.
452          */
453         do {
454                 *dst++ = *src++;
455         } while (--cnt != 0);
456 #endif
457 }
458
459 /*
460  * Linear Interpolation. This at least sounds better (perceptually) and fast,
461  * but without any proper filtering which means aliasing still exist and
462  * could become worst with a right sample. Interpolation centered within
463  * Z_LINEAR_ONE between the present and previous sample and everything is
464  * done with simple 32bit scaling arithmetic.
465  */
466 #define Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                                     \
467 static void                                                                     \
468 z_feed_linear_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)            \
469 {                                                                               \
470         int32_t z;                                                              \
471         intpcm_t x, y;                                                          \
472         uint32_t ch;                                                            \
473         uint8_t *sx, *sy;                                                       \
474                                                                                 \
475         z = ((uint32_t)info->z_alpha * info->z_dx) >> Z_LINEAR_UNSHIFT;         \
476                                                                                 \
477         sx = info->z_delay + (info->z_start * info->channels *                  \
478             PCM_##BIT##_BPS);                                                   \
479         sy = sx - (info->channels * PCM_##BIT##_BPS);                           \
480                                                                                 \
481         ch = info->channels;                                                    \
482                                                                                 \
483         do {                                                                    \
484                 x = _PCM_READ_##SIGN##BIT##_##ENDIAN(sx);                       \
485                 y = _PCM_READ_##SIGN##BIT##_##ENDIAN(sy);                       \
486                 x = Z_LINEAR_INTERPOLATE_##BIT(z, x, y);                        \
487                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, x);                      \
488                 sx += PCM_##BIT##_BPS;                                          \
489                 sy += PCM_##BIT##_BPS;                                          \
490                 dst += PCM_##BIT##_BPS;                                         \
491         } while (--ch != 0);                                                    \
492 }
493
494 /*
495  * Userland clipping diagnostic check, not enabled in kernel compilation.
496  * While doing sinc interpolation, unrealistic samples like full scale sine
497  * wav will clip, but for other things this will not make any noise at all.
498  * Everybody should learn how to normalized perceived loudness of their own
499  * music/sounds/samples (hint: ReplayGain).
500  */
501 #ifdef Z_DIAGNOSTIC
502 #define Z_CLIP_CHECK(v, BIT)    do {                                    \
503         if ((v) > PCM_S##BIT##_MAX) {                                   \
504                 fprintf(stderr, "Overflow: v=%jd, max=%jd\n",           \
505                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MAX);         \
506         } else if ((v) < PCM_S##BIT##_MIN) {                            \
507                 fprintf(stderr, "Underflow: v=%jd, min=%jd\n",          \
508                     (intmax_t)(v), (intmax_t)PCM_S##BIT##_MIN);         \
509         }                                                               \
510 } while (0)
511 #else
512 #define Z_CLIP_CHECK(...)
513 #endif
514
515 #define Z_CLAMP(v, BIT)                                                 \
516         (((v) > PCM_S##BIT##_MAX) ? PCM_S##BIT##_MAX :                  \
517         (((v) < PCM_S##BIT##_MIN) ? PCM_S##BIT##_MIN : (v)))
518
519 /*
520  * Sine Cardinal (SINC) Interpolation. Scaling is done in 64 bit, so
521  * there's no point to hold the plate any longer. All samples will be
522  * shifted to a full 32 bit, scaled and restored during write for
523  * maximum dynamic range (only for downsampling).
524  */
525 #define _Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, adv)                      \
526         c += z >> Z_SHIFT;                                              \
527         z &= Z_MASK;                                                    \
528         coeff = Z_COEFF_INTERPOLATE(z, z_coeff[c], z_dcoeff[c]);        \
529         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                        \
530         v += Z_NORM_##BIT((intpcm64_t)x * coeff);                       \
531         z += info->z_dy;                                                \
532         p adv##= info->channels * PCM_##BIT##_BPS
533
534 /* 
535  * XXX GCC4 optimization is such a !@#$%, need manual unrolling.
536  */
537 #if defined(__GNUC__) && __GNUC__ >= 4
538 #define Z_SINC_ACCUMULATE(...)  do {                                    \
539         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
540         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
541 } while (0)
542 #define Z_SINC_ACCUMULATE_DECR          2
543 #else
544 #define Z_SINC_ACCUMULATE(...)  do {                                    \
545         _Z_SINC_ACCUMULATE(__VA_ARGS__);                                \
546 } while (0)
547 #define Z_SINC_ACCUMULATE_DECR          1
548 #endif
549
550 #define Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                                       \
551 static void                                                                     \
552 z_feed_sinc_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)              \
553 {                                                                               \
554         intpcm64_t v;                                                           \
555         intpcm_t x;                                                             \
556         uint8_t *p;                                                             \
557         int32_t coeff, z, *z_coeff, *z_dcoeff;                                  \
558         uint32_t c, center, ch, i;                                              \
559                                                                                 \
560         z_coeff = info->z_coeff;                                                \
561         z_dcoeff = info->z_dcoeff;                                              \
562         center = z_prev(info, info->z_start, info->z_size);                     \
563         ch = info->channels * PCM_##BIT##_BPS;                                  \
564         dst += ch;                                                              \
565                                                                                 \
566         do {                                                                    \
567                 dst -= PCM_##BIT##_BPS;                                         \
568                 ch -= PCM_##BIT##_BPS;                                          \
569                 v = 0;                                                          \
570                 z = info->z_alpha * info->z_dx;                                 \
571                 c = 0;                                                          \
572                 p = info->z_delay + (z_next(info, center, 1) *                  \
573                     info->channels * PCM_##BIT##_BPS) + ch;                     \
574                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
575                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, +);                \
576                 z = info->z_dy - (info->z_alpha * info->z_dx);                  \
577                 c = 0;                                                          \
578                 p = info->z_delay + (center * info->channels *                  \
579                     PCM_##BIT##_BPS) + ch;                                      \
580                 for (i = info->z_size; i != 0; i -= Z_SINC_ACCUMULATE_DECR)     \
581                         Z_SINC_ACCUMULATE(SIGN, BIT, ENDIAN, -);                \
582                 if (info->z_scale != Z_ONE)                                     \
583                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
584                 else                                                            \
585                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
586                 Z_CLIP_CHECK(v, BIT);                                           \
587                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
588         } while (ch != 0);                                                      \
589 }
590
591 #define Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)                             \
592 static void                                                                     \
593 z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN(struct z_info *info, uint8_t *dst)    \
594 {                                                                               \
595         intpcm64_t v;                                                           \
596         intpcm_t x;                                                             \
597         uint8_t *p;                                                             \
598         int32_t ch, i, start, *z_pcoeff;                                        \
599                                                                                 \
600         ch = info->channels * PCM_##BIT##_BPS;                                  \
601         dst += ch;                                                              \
602         start = z_prev(info, info->z_start, (info->z_size << 1) - 1) * ch;      \
603                                                                                 \
604         do {                                                                    \
605                 dst -= PCM_##BIT##_BPS;                                         \
606                 ch -= PCM_##BIT##_BPS;                                          \
607                 v = 0;                                                          \
608                 p = info->z_delay + start + ch;                                 \
609                 z_pcoeff = info->z_pcoeff +                                     \
610                     ((info->z_alpha * info->z_size) << 1);                      \
611                 for (i = info->z_size; i != 0; i--) {                           \
612                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
613                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
614                         z_pcoeff++;                                             \
615                         p += info->channels * PCM_##BIT##_BPS;                  \
616                         x = _PCM_READ_##SIGN##BIT##_##ENDIAN(p);                \
617                         v += Z_NORM_##BIT((intpcm64_t)x * *z_pcoeff);           \
618                         z_pcoeff++;                                             \
619                         p += info->channels * PCM_##BIT##_BPS;                  \
620                 }                                                               \
621                 if (info->z_scale != Z_ONE)                                     \
622                         v = Z_SCALE_##BIT(v, info->z_scale);                    \
623                 else                                                            \
624                         v >>= Z_COEFF_SHIFT - Z_GUARD_BIT_##BIT;                \
625                 Z_CLIP_CHECK(v, BIT);                                           \
626                 _PCM_WRITE_##SIGN##BIT##_##ENDIAN(dst, Z_CLAMP(v, BIT));        \
627         } while (ch != 0);                                                      \
628 }
629
630 #define Z_DECLARE(SIGN, BIT, ENDIAN)                                    \
631         Z_DECLARE_LINEAR(SIGN, BIT, ENDIAN)                             \
632         Z_DECLARE_SINC(SIGN, BIT, ENDIAN)                               \
633         Z_DECLARE_SINC_POLYPHASE(SIGN, BIT, ENDIAN)
634
635 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
636 Z_DECLARE(S, 16, LE)
637 Z_DECLARE(S, 32, LE)
638 #endif
639 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
640 Z_DECLARE(S, 16, BE)
641 Z_DECLARE(S, 32, BE)
642 #endif
643 #ifdef SND_FEEDER_MULTIFORMAT
644 Z_DECLARE(S,  8, NE)
645 Z_DECLARE(S, 24, LE)
646 Z_DECLARE(S, 24, BE)
647 Z_DECLARE(U,  8, NE)
648 Z_DECLARE(U, 16, LE)
649 Z_DECLARE(U, 24, LE)
650 Z_DECLARE(U, 32, LE)
651 Z_DECLARE(U, 16, BE)
652 Z_DECLARE(U, 24, BE)
653 Z_DECLARE(U, 32, BE)
654 #endif
655
656 enum {
657         Z_RESAMPLER_ZOH,
658         Z_RESAMPLER_LINEAR,
659         Z_RESAMPLER_SINC,
660         Z_RESAMPLER_SINC_POLYPHASE,
661         Z_RESAMPLER_LAST
662 };
663
664 #define Z_RESAMPLER_IDX(i)                                              \
665         (Z_IS_SINC(i) ? Z_RESAMPLER_SINC : (i)->quality)
666
667 #define Z_RESAMPLER_ENTRY(SIGN, BIT, ENDIAN)                                    \
668         {                                                                       \
669             AFMT_##SIGN##BIT##_##ENDIAN,                                        \
670             {                                                                   \
671                 [Z_RESAMPLER_ZOH]    = z_feed_zoh,                              \
672                 [Z_RESAMPLER_LINEAR] = z_feed_linear_##SIGN##BIT##ENDIAN,       \
673                 [Z_RESAMPLER_SINC]   = z_feed_sinc_##SIGN##BIT##ENDIAN,         \
674                 [Z_RESAMPLER_SINC_POLYPHASE]   =                                \
675                     z_feed_sinc_polyphase_##SIGN##BIT##ENDIAN                   \
676             }                                                                   \
677         }
678
679 static const struct {
680         uint32_t format;
681         z_resampler_t resampler[Z_RESAMPLER_LAST];
682 } z_resampler_tab[] = {
683 #if BYTE_ORDER == LITTLE_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
684         Z_RESAMPLER_ENTRY(S, 16, LE),
685         Z_RESAMPLER_ENTRY(S, 32, LE),
686 #endif
687 #if BYTE_ORDER == BIG_ENDIAN || defined(SND_FEEDER_MULTIFORMAT)
688         Z_RESAMPLER_ENTRY(S, 16, BE),
689         Z_RESAMPLER_ENTRY(S, 32, BE),
690 #endif
691 #ifdef SND_FEEDER_MULTIFORMAT
692         Z_RESAMPLER_ENTRY(S,  8, NE),
693         Z_RESAMPLER_ENTRY(S, 24, LE),
694         Z_RESAMPLER_ENTRY(S, 24, BE),
695         Z_RESAMPLER_ENTRY(U,  8, NE),
696         Z_RESAMPLER_ENTRY(U, 16, LE),
697         Z_RESAMPLER_ENTRY(U, 24, LE),
698         Z_RESAMPLER_ENTRY(U, 32, LE),
699         Z_RESAMPLER_ENTRY(U, 16, BE),
700         Z_RESAMPLER_ENTRY(U, 24, BE),
701         Z_RESAMPLER_ENTRY(U, 32, BE),
702 #endif
703 };
704
705 #define Z_RESAMPLER_TAB_SIZE                                            \
706         ((int32_t)(sizeof(z_resampler_tab) / sizeof(z_resampler_tab[0])))
707
708 static void
709 z_resampler_reset(struct z_info *info)
710 {
711
712         info->src = info->rsrc - (info->rsrc % ((feeder_rate_round > 0 &&
713             info->rsrc > feeder_rate_round) ? feeder_rate_round : 1));
714         info->dst = info->rdst - (info->rdst % ((feeder_rate_round > 0 &&
715             info->rdst > feeder_rate_round) ? feeder_rate_round : 1));
716         info->z_gx = 1;
717         info->z_gy = 1;
718         info->z_alpha = 0;
719         info->z_resample = NULL;
720         info->z_size = 1;
721         info->z_coeff = NULL;
722         info->z_dcoeff = NULL;
723         if (info->z_pcoeff != NULL) {
724                 kfree(info->z_pcoeff, M_DEVBUF);
725                 info->z_pcoeff = NULL;
726         }
727         info->z_scale = Z_ONE;
728         info->z_dx = Z_FULL_ONE;
729         info->z_dy = Z_FULL_ONE;
730 #ifdef Z_DIAGNOSTIC
731         info->z_cycle = 0;
732 #endif
733         if (info->quality < Z_QUALITY_MIN)
734                 info->quality = Z_QUALITY_MIN;
735         else if (info->quality > Z_QUALITY_MAX)
736                 info->quality = Z_QUALITY_MAX;
737 }
738
739 #ifdef Z_PARANOID
740 static int32_t
741 z_resampler_sinc_len(struct z_info *info)
742 {
743         int32_t c, z, len, lmax;
744
745         if (!Z_IS_SINC(info))
746                 return (1);
747
748         /*
749          * A rather careful (or useless) way to calculate filter length.
750          * Z_SINC_LEN() itself is accurate enough to do its job. Extra
751          * sanity checking is not going to hurt though..
752          */
753         c = 0;
754         z = info->z_dy;
755         len = 0;
756         lmax = z_coeff_tab[Z_SINC_COEFF_IDX(info)].len;
757
758         do {
759                 c += z >> Z_SHIFT;
760                 z &= Z_MASK;
761                 z += info->z_dy;
762         } while (c < lmax && ++len > 0);
763
764         if (len != Z_SINC_LEN(info)) {
765 #ifdef _KERNEL
766                 kprintf("%s(): sinc l=%d != Z_SINC_LEN=%d\n",
767                     __func__, len, Z_SINC_LEN(info));
768 #else
769                 fprintf(stderr, "%s(): sinc l=%d != Z_SINC_LEN=%d\n",
770                     __func__, len, Z_SINC_LEN(info));
771                 return (-1);
772 #endif
773         }
774
775         return (len);
776 }
777 #else
778 #define z_resampler_sinc_len(i)         (Z_IS_SINC(i) ? Z_SINC_LEN(i) : 1)
779 #endif
780
781 #define Z_POLYPHASE_COEFF_SHIFT         0
782
783 /*
784  * Pick suitable polynomial interpolators based on filter oversampled ratio
785  * (2 ^ Z_DRIFT_SHIFT).
786  */
787 #if !(defined(Z_COEFF_INTERP_ZOH) || defined(Z_COEFF_INTERP_LINEAR) ||          \
788     defined(Z_COEFF_INTERP_QUADRATIC) || defined(Z_COEFF_INTERP_HERMITE) ||     \
789     defined(Z_COEFF_INTER_BSPLINE) || defined(Z_COEFF_INTERP_OPT32X) ||         \
790     defined(Z_COEFF_INTERP_OPT16X) || defined(Z_COEFF_INTERP_OPT8X) ||          \
791     defined(Z_COEFF_INTERP_OPT4X) || defined(Z_COEFF_INTERP_OPT2X))
792 #if Z_DRIFT_SHIFT >= 6
793 #define Z_COEFF_INTERP_BSPLINE          1
794 #elif Z_DRIFT_SHIFT >= 5
795 #define Z_COEFF_INTERP_OPT32X           1
796 #elif Z_DRIFT_SHIFT == 4
797 #define Z_COEFF_INTERP_OPT16X           1
798 #elif Z_DRIFT_SHIFT == 3
799 #define Z_COEFF_INTERP_OPT8X            1
800 #elif Z_DRIFT_SHIFT == 2
801 #define Z_COEFF_INTERP_OPT4X            1
802 #elif Z_DRIFT_SHIFT == 1
803 #define Z_COEFF_INTERP_OPT2X            1
804 #else
805 #error "Z_DRIFT_SHIFT screwed!"
806 #endif
807 #endif
808
809 /*
810  * In classic polyphase mode, the actual coefficients for each phases need to
811  * be calculated based on default prototype filters. For highly oversampled
812  * filter, linear or quadradatic interpolator should be enough. Anything less
813  * than that require 'special' interpolators to reduce interpolation errors.
814  *
815  * "Polynomial Interpolators for High-Quality Resampling of Oversampled Audio"
816  *    by Olli Niemitalo
817  *    - http://www.student.oulu.fi/~oniemita/dsp/deip.pdf
818  *
819  */
820 static int32_t
821 z_coeff_interpolate(int32_t z, int32_t *z_coeff)
822 {
823         int32_t coeff;
824 #if defined(Z_COEFF_INTERP_ZOH)
825
826         /* 1-point, 0th-order (Zero Order Hold) */
827         z = z;
828         coeff = z_coeff[0];
829 #elif defined(Z_COEFF_INTERP_LINEAR)
830         int32_t zl0, zl1;
831
832         /* 2-point, 1st-order Linear */
833         zl0 = z_coeff[0];
834         zl1 = z_coeff[1] - z_coeff[0];
835
836         coeff = Z_RSHIFT((int64_t)zl1 * z, Z_SHIFT) + zl0;
837 #elif defined(Z_COEFF_INTERP_QUADRATIC)
838         int32_t zq0, zq1, zq2;
839
840         /* 3-point, 2nd-order Quadratic */
841         zq0 = z_coeff[0];
842         zq1 = z_coeff[1] - z_coeff[-1];
843         zq2 = z_coeff[1] + z_coeff[-1] - (z_coeff[0] << 1);
844
845         coeff = Z_RSHIFT((Z_RSHIFT((int64_t)zq2 * z, Z_SHIFT) +
846             zq1) * z, Z_SHIFT + 1) + zq0;
847 #elif defined(Z_COEFF_INTERP_HERMITE)
848         int32_t zh0, zh1, zh2, zh3;
849
850         /* 4-point, 3rd-order Hermite */
851         zh0 = z_coeff[0];
852         zh1 = z_coeff[1] - z_coeff[-1];
853         zh2 = (z_coeff[-1] << 1) - (z_coeff[0] * 5) + (z_coeff[1] << 2) -
854             z_coeff[2];
855         zh3 = z_coeff[2] - z_coeff[-1] + ((z_coeff[0] - z_coeff[1]) * 3);
856
857         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zh3 * z, Z_SHIFT) +
858             zh2) * z, Z_SHIFT) + zh1) * z, Z_SHIFT + 1) + zh0;
859 #elif defined(Z_COEFF_INTERP_BSPLINE)
860         int32_t zb0, zb1, zb2, zb3;
861
862         /* 4-point, 3rd-order B-Spline */
863         zb0 = Z_RSHIFT(0x15555555LL * (((int64_t)z_coeff[0] << 2) +
864             z_coeff[-1] + z_coeff[1]), 30);
865         zb1 = z_coeff[1] - z_coeff[-1];
866         zb2 = z_coeff[-1] + z_coeff[1] - (z_coeff[0] << 1);
867         zb3 = Z_RSHIFT(0x15555555LL * (((z_coeff[0] - z_coeff[1]) * 3) +
868             z_coeff[2] - z_coeff[-1]), 30);
869
870         coeff = (Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((int64_t)zb3 * z, Z_SHIFT) +
871             zb2) * z, Z_SHIFT) + zb1) * z, Z_SHIFT) + zb0 + 1) >> 1;
872 #elif defined(Z_COEFF_INTERP_OPT32X)
873         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
874         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
875
876         /* 6-point, 5th-order Optimal 32x */
877         zoz = z - (Z_ONE >> 1);
878         zoe1 = z_coeff[1] + z_coeff[0];
879         zoe2 = z_coeff[2] + z_coeff[-1];
880         zoe3 = z_coeff[3] + z_coeff[-2];
881         zoo1 = z_coeff[1] - z_coeff[0];
882         zoo2 = z_coeff[2] - z_coeff[-1];
883         zoo3 = z_coeff[3] - z_coeff[-2];
884
885         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
886             (0x00170c29LL * zoe3), 30);
887         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
888             (0x008cd4dcLL * zoo3), 30);
889         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
890             (0x0160b5d0LL * zoe3), 30);
891         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
892             (0x01cfe914LL * zoo3), 30);
893         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
894             (0x015508ddLL * zoe3), 30);
895         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
896             (0x0082d81aLL * zoo3), 30);
897
898         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
899             (int64_t)zoc5 * zoz, Z_SHIFT) +
900             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
901             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
902 #elif defined(Z_COEFF_INTERP_OPT16X)
903         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
904         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
905
906         /* 6-point, 5th-order Optimal 16x */
907         zoz = z - (Z_ONE >> 1);
908         zoe1 = z_coeff[1] + z_coeff[0];
909         zoe2 = z_coeff[2] + z_coeff[-1];
910         zoe3 = z_coeff[3] + z_coeff[-2];
911         zoo1 = z_coeff[1] - z_coeff[0];
912         zoo2 = z_coeff[2] - z_coeff[-1];
913         zoo3 = z_coeff[3] - z_coeff[-2];
914
915         zoc0 = Z_RSHIFT((0x1ac2260dLL * zoe1) + (0x0526cdcaLL * zoe2) +
916             (0x00170c29LL * zoe3), 30);
917         zoc1 = Z_RSHIFT((0x14f8a49aLL * zoo1) + (0x0d6d1109LL * zoo2) +
918             (0x008cd4dcLL * zoo3), 30);
919         zoc2 = Z_RSHIFT((-0x0d3e94a4LL * zoe1) + (0x0bddded4LL * zoe2) +
920             (0x0160b5d0LL * zoe3), 30);
921         zoc3 = Z_RSHIFT((-0x0de10cc4LL * zoo1) + (0x019b2a7dLL * zoo2) +
922             (0x01cfe914LL * zoo3), 30);
923         zoc4 = Z_RSHIFT((0x02aa12d7LL * zoe1) + (-0x03ff1bb3LL * zoe2) +
924             (0x015508ddLL * zoe3), 30);
925         zoc5 = Z_RSHIFT((0x051d29e5LL * zoo1) + (-0x028e7647LL * zoo2) +
926             (0x0082d81aLL * zoo3), 30);
927
928         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
929             (int64_t)zoc5 * zoz, Z_SHIFT) +
930             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
931             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
932 #elif defined(Z_COEFF_INTERP_OPT8X)
933         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
934         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
935
936         /* 6-point, 5th-order Optimal 8x */
937         zoz = z - (Z_ONE >> 1);
938         zoe1 = z_coeff[1] + z_coeff[0];
939         zoe2 = z_coeff[2] + z_coeff[-1];
940         zoe3 = z_coeff[3] + z_coeff[-2];
941         zoo1 = z_coeff[1] - z_coeff[0];
942         zoo2 = z_coeff[2] - z_coeff[-1];
943         zoo3 = z_coeff[3] - z_coeff[-2];
944
945         zoc0 = Z_RSHIFT((0x1aa9b47dLL * zoe1) + (0x053d9944LL * zoe2) +
946             (0x0018b23fLL * zoe3), 30);
947         zoc1 = Z_RSHIFT((0x14a104d1LL * zoo1) + (0x0d7d2504LL * zoo2) +
948             (0x0094b599LL * zoo3), 30);
949         zoc2 = Z_RSHIFT((-0x0d22530bLL * zoe1) + (0x0bb37a2cLL * zoe2) +
950             (0x016ed8e0LL * zoe3), 30);
951         zoc3 = Z_RSHIFT((-0x0d744b1cLL * zoo1) + (0x01649591LL * zoo2) +
952             (0x01dae93aLL * zoo3), 30);
953         zoc4 = Z_RSHIFT((0x02a7ee1bLL * zoe1) + (-0x03fbdb24LL * zoe2) +
954             (0x0153ed07LL * zoe3), 30);
955         zoc5 = Z_RSHIFT((0x04cf9b6cLL * zoo1) + (-0x0266b378LL * zoo2) +
956             (0x007a7c26LL * zoo3), 30);
957
958         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
959             (int64_t)zoc5 * zoz, Z_SHIFT) +
960             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
961             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
962 #elif defined(Z_COEFF_INTERP_OPT4X)
963         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
964         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
965
966         /* 6-point, 5th-order Optimal 4x */
967         zoz = z - (Z_ONE >> 1);
968         zoe1 = z_coeff[1] + z_coeff[0];
969         zoe2 = z_coeff[2] + z_coeff[-1];
970         zoe3 = z_coeff[3] + z_coeff[-2];
971         zoo1 = z_coeff[1] - z_coeff[0];
972         zoo2 = z_coeff[2] - z_coeff[-1];
973         zoo3 = z_coeff[3] - z_coeff[-2];
974
975         zoc0 = Z_RSHIFT((0x1a8eda43LL * zoe1) + (0x0556ee38LL * zoe2) +
976             (0x001a3784LL * zoe3), 30);
977         zoc1 = Z_RSHIFT((0x143d863eLL * zoo1) + (0x0d910e36LL * zoo2) +
978             (0x009ca889LL * zoo3), 30);
979         zoc2 = Z_RSHIFT((-0x0d026821LL * zoe1) + (0x0b837773LL * zoe2) +
980             (0x017ef0c6LL * zoe3), 30);
981         zoc3 = Z_RSHIFT((-0x0cef1502LL * zoo1) + (0x01207a8eLL * zoo2) +
982             (0x01e936dbLL * zoo3), 30);
983         zoc4 = Z_RSHIFT((0x029fe643LL * zoe1) + (-0x03ef3fc8LL * zoe2) +
984             (0x014f5923LL * zoe3), 30);
985         zoc5 = Z_RSHIFT((0x043a9d08LL * zoo1) + (-0x02154febLL * zoo2) +
986             (0x00670dbdLL * zoo3), 30);
987
988         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
989             (int64_t)zoc5 * zoz, Z_SHIFT) +
990             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
991             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
992 #elif defined(Z_COEFF_INTERP_OPT2X)
993         int32_t zoz, zoe1, zoe2, zoe3, zoo1, zoo2, zoo3;
994         int32_t zoc0, zoc1, zoc2, zoc3, zoc4, zoc5;
995
996         /* 6-point, 5th-order Optimal 2x */
997         zoz = z - (Z_ONE >> 1);
998         zoe1 = z_coeff[1] + z_coeff[0];
999         zoe2 = z_coeff[2] + z_coeff[-1];
1000         zoe3 = z_coeff[3] + z_coeff[-2];
1001         zoo1 = z_coeff[1] - z_coeff[0];
1002         zoo2 = z_coeff[2] - z_coeff[-1];
1003         zoo3 = z_coeff[3] - z_coeff[-2];
1004
1005         zoc0 = Z_RSHIFT((0x19edb6fdLL * zoe1) + (0x05ebd062LL * zoe2) +
1006             (0x00267881LL * zoe3), 30);
1007         zoc1 = Z_RSHIFT((0x1223af76LL * zoo1) + (0x0de3dd6bLL * zoo2) +
1008             (0x00d683cdLL * zoo3), 30);
1009         zoc2 = Z_RSHIFT((-0x0c3ee068LL * zoe1) + (0x0a5c3769LL * zoe2) +
1010             (0x01e2aceaLL * zoe3), 30);
1011         zoc3 = Z_RSHIFT((-0x0a8ab614LL * zoo1) + (-0x0019522eLL * zoo2) +
1012             (0x022cefc7LL * zoo3), 30);
1013         zoc4 = Z_RSHIFT((0x0276187dLL * zoe1) + (-0x03a801e8LL * zoe2) +
1014             (0x0131d935LL * zoe3), 30);
1015         zoc5 = Z_RSHIFT((0x02c373f5LL * zoo1) + (-0x01275f83LL * zoo2) +
1016             (0x0018ee79LL * zoo3), 30);
1017
1018         coeff = Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT((Z_RSHIFT(
1019             (int64_t)zoc5 * zoz, Z_SHIFT) +
1020             zoc4) * zoz, Z_SHIFT) + zoc3) * zoz, Z_SHIFT) +
1021             zoc2) * zoz, Z_SHIFT) + zoc1) * zoz, Z_SHIFT) + zoc0;
1022 #else
1023 #error "Interpolation type screwed!"
1024 #endif
1025
1026 #if Z_POLYPHASE_COEFF_SHIFT > 0
1027         coeff = Z_RSHIFT(coeff, Z_POLYPHASE_COEFF_SHIFT);
1028 #endif
1029         return (coeff);
1030 }
1031
1032 static int
1033 z_resampler_build_polyphase(struct z_info *info)
1034 {
1035         int32_t alpha, c, i, z, idx;
1036
1037         /* Let this be here first. */
1038         if (info->z_pcoeff != NULL) {
1039                 kfree(info->z_pcoeff, M_DEVBUF);
1040                 info->z_pcoeff = NULL;
1041         }
1042
1043         if (feeder_rate_polyphase_max < 1)
1044                 return (ENOTSUP);
1045
1046         if (((int64_t)info->z_size * info->z_gy * 2) >
1047             feeder_rate_polyphase_max) {
1048 #ifndef _KERNEL
1049                 fprintf(stderr, "Polyphase entries exceed: [%d/%d] %jd > %d\n",
1050                     info->z_gx, info->z_gy,
1051                     (intmax_t)info->z_size * info->z_gy * 2,
1052                     feeder_rate_polyphase_max);
1053 #endif
1054                 return (E2BIG);
1055         }
1056
1057         info->z_pcoeff = kmalloc(sizeof(int32_t) *
1058             info->z_size * info->z_gy * 2, M_DEVBUF, M_WAITOK | M_ZERO);
1059         if (info->z_pcoeff == NULL)
1060                 return (ENOMEM);
1061
1062         for (alpha = 0; alpha < info->z_gy; alpha++) {
1063                 z = alpha * info->z_dx;
1064                 c = 0;
1065                 for (i = info->z_size; i != 0; i--) {
1066                         c += z >> Z_SHIFT;
1067                         z &= Z_MASK;
1068                         idx = (alpha * info->z_size * 2) +
1069                             (info->z_size * 2) - i;
1070                         info->z_pcoeff[idx] =
1071                             z_coeff_interpolate(z, info->z_coeff + c);
1072                         z += info->z_dy;
1073                 }
1074                 z = info->z_dy - (alpha * info->z_dx);
1075                 c = 0;
1076                 for (i = info->z_size; i != 0; i--) {
1077                         c += z >> Z_SHIFT;
1078                         z &= Z_MASK;
1079                         idx = (alpha * info->z_size * 2) + i - 1;
1080                         info->z_pcoeff[idx] =
1081                             z_coeff_interpolate(z, info->z_coeff + c);
1082                         z += info->z_dy;
1083                 }
1084         }
1085         
1086 #ifndef _KERNEL
1087         fprintf(stderr, "Polyphase: [%d/%d] %d entries\n",
1088             info->z_gx, info->z_gy, info->z_size * info->z_gy * 2);
1089 #endif
1090
1091         return (0);
1092 }
1093
1094 static int
1095 z_resampler_setup(struct pcm_feeder *f)
1096 {
1097         struct z_info *info;
1098         int64_t gy2gx_max, gx2gy_max;
1099         uint32_t format;
1100         int32_t align, i, z_scale;
1101         int adaptive;
1102
1103         info = f->data;
1104         z_resampler_reset(info);
1105
1106         if (info->src == info->dst)
1107                 return (0);
1108
1109         /* Shrink by greatest common divisor. */
1110         i = z_gcd(info->src, info->dst);
1111         info->z_gx = info->src / i;
1112         info->z_gy = info->dst / i;
1113
1114         /* Too big, or too small. Bail out. */
1115         if (!(Z_FACTOR_SAFE(info->z_gx) && Z_FACTOR_SAFE(info->z_gy)))
1116                 return (EINVAL);
1117
1118         format = f->desc->in;
1119         adaptive = 0;
1120         z_scale = 0;
1121
1122         /*
1123          * Setup everything: filter length, conversion factor, etc.
1124          */
1125         if (Z_IS_SINC(info)) {
1126                 /*
1127                  * Downsampling, or upsampling scaling factor. As long as the
1128                  * factor can be represented by a fraction of 1 << Z_SHIFT,
1129                  * we're pretty much in business. Scaling is not needed for
1130                  * upsampling, so we just slap Z_ONE there.
1131                  */
1132                 if (info->z_gx > info->z_gy)
1133                         /*
1134                          * If the downsampling ratio is beyond sanity,
1135                          * enable semi-adaptive mode. Although handling
1136                          * extreme ratio is possible, the result of the
1137                          * conversion is just pointless, unworthy,
1138                          * nonsensical noises, etc.
1139                          */
1140                         if ((info->z_gx / info->z_gy) > Z_SINC_DOWNMAX)
1141                                 z_scale = Z_ONE / Z_SINC_DOWNMAX;
1142                         else
1143                                 z_scale = ((uint64_t)info->z_gy << Z_SHIFT) /
1144                                     info->z_gx;
1145                 else
1146                         z_scale = Z_ONE;
1147
1148                 /*
1149                  * This is actually impossible, unless anything above
1150                  * overflow.
1151                  */
1152                 if (z_scale < 1)
1153                         return (E2BIG);
1154
1155                 /*
1156                  * Calculate sample time/coefficients index drift. It is
1157                  * a constant for upsampling, but downsampling require
1158                  * heavy duty filtering with possible too long filters.
1159                  * If anything goes wrong, revisit again and enable
1160                  * adaptive mode.
1161                  */
1162 z_setup_adaptive_sinc:
1163                 if (info->z_pcoeff != NULL) {
1164                         kfree(info->z_pcoeff, M_DEVBUF);
1165                         info->z_pcoeff = NULL;
1166                 }
1167
1168                 if (adaptive == 0) {
1169                         info->z_dy = z_scale << Z_DRIFT_SHIFT;
1170                         if (info->z_dy < 1)
1171                                 return (E2BIG);
1172                         info->z_scale = z_scale;
1173                 } else {
1174                         info->z_dy = Z_FULL_ONE;
1175                         info->z_scale = Z_ONE;
1176                 }
1177
1178 #if 0
1179 #define Z_SCALE_DIV     10000
1180 #define Z_SCALE_LIMIT(s, v)                                             \
1181         ((((uint64_t)(s) * (v)) + (Z_SCALE_DIV >> 1)) / Z_SCALE_DIV)
1182
1183                 info->z_scale = Z_SCALE_LIMIT(info->z_scale, 9780);
1184 #endif
1185
1186                 /* Smallest drift increment. */
1187                 info->z_dx = info->z_dy / info->z_gy;
1188
1189                 /*
1190                  * Overflow or underflow. Try adaptive, let it continue and
1191                  * retry.
1192                  */
1193                 if (info->z_dx < 1) {
1194                         if (adaptive == 0) {
1195                                 adaptive = 1;
1196                                 goto z_setup_adaptive_sinc;
1197                         }
1198                         return (E2BIG);
1199                 }
1200
1201                 /*
1202                  * Round back output drift.
1203                  */
1204                 info->z_dy = info->z_dx * info->z_gy;
1205
1206                 for (i = 0; i < Z_COEFF_TAB_SIZE; i++) {
1207                         if (Z_SINC_COEFF_IDX(info) != i)
1208                                 continue;
1209                         /*
1210                          * Calculate required filter length and guard
1211                          * against possible abusive result. Note that
1212                          * this represents only 1/2 of the entire filter
1213                          * length.
1214                          */
1215                         info->z_size = z_resampler_sinc_len(info);
1216
1217                         /*
1218                          * Multiple of 2 rounding, for better accumulator
1219                          * performance.
1220                          */
1221                         info->z_size &= ~1;
1222
1223                         if (info->z_size < 2 || info->z_size > Z_SINC_MAX) {
1224                                 if (adaptive == 0) {
1225                                         adaptive = 1;
1226                                         goto z_setup_adaptive_sinc;
1227                                 }
1228                                 return (E2BIG);
1229                         }
1230                         info->z_coeff = z_coeff_tab[i].coeff + Z_COEFF_OFFSET;
1231                         info->z_dcoeff = z_coeff_tab[i].dcoeff;
1232                         break;
1233                 }
1234
1235                 if (info->z_coeff == NULL || info->z_dcoeff == NULL)
1236                         return (EINVAL);
1237         } else if (Z_IS_LINEAR(info)) {
1238                 /*
1239                  * Don't put much effort if we're doing linear interpolation.
1240                  * Just center the interpolation distance within Z_LINEAR_ONE,
1241                  * and be happy about it.
1242                  */
1243                 info->z_dx = Z_LINEAR_FULL_ONE / info->z_gy;
1244         }
1245
1246         /*
1247          * We're safe for now, lets continue.. Look for our resampler
1248          * depending on configured format and quality.
1249          */
1250         for (i = 0; i < Z_RESAMPLER_TAB_SIZE; i++) {
1251                 int ridx;
1252
1253                 if (AFMT_ENCODING(format) != z_resampler_tab[i].format)
1254                         continue;
1255                 if (Z_IS_SINC(info) && adaptive == 0 &&
1256                     z_resampler_build_polyphase(info) == 0)
1257                         ridx = Z_RESAMPLER_SINC_POLYPHASE;
1258                 else
1259                         ridx = Z_RESAMPLER_IDX(info);
1260                 info->z_resample = z_resampler_tab[i].resampler[ridx];
1261                 break;
1262         }
1263
1264         if (info->z_resample == NULL)
1265                 return (EINVAL);
1266
1267         info->bps = AFMT_BPS(format);
1268         align = info->channels * info->bps;
1269
1270         /*
1271          * Calculate largest value that can be fed into z_gy2gx() and
1272          * z_gx2gy() without causing (signed) 32bit overflow. z_gy2gx() will
1273          * be called early during feeding process to determine how much input
1274          * samples that is required to generate requested output, while
1275          * z_gx2gy() will be called just before samples filtering /
1276          * accumulation process based on available samples that has been
1277          * calculated using z_gx2gy().
1278          *
1279          * Now that is damn confusing, I guess ;-) .
1280          */
1281         gy2gx_max = (((uint64_t)info->z_gy * INT32_MAX) - info->z_gy + 1) /
1282             info->z_gx;
1283
1284         if ((gy2gx_max * align) > SND_FXDIV_MAX)
1285                 gy2gx_max = SND_FXDIV_MAX / align;
1286
1287         if (gy2gx_max < 1)
1288                 return (E2BIG);
1289
1290         gx2gy_max = (((uint64_t)info->z_gx * INT32_MAX) - info->z_gy) /
1291             info->z_gy;
1292
1293         if (gx2gy_max > INT32_MAX)
1294                 gx2gy_max = INT32_MAX;
1295
1296         if (gx2gy_max < 1)
1297                 return (E2BIG);
1298
1299         /*
1300          * Ensure that z_gy2gx() at its largest possible calculated value
1301          * (alpha = 0) will not cause overflow further late during z_gx2gy()
1302          * stage.
1303          */
1304         if (z_gy2gx(info, gy2gx_max) > _Z_GCAST(gx2gy_max))
1305                 return (E2BIG);
1306
1307         info->z_maxfeed = gy2gx_max * align;
1308
1309 #ifdef Z_USE_ALPHADRIFT
1310         info->z_startdrift = z_gy2gx(info, 1);
1311         info->z_alphadrift = z_drift(info, info->z_startdrift, 1);
1312 #endif
1313
1314         i = z_gy2gx(info, 1);
1315         info->z_full = z_roundpow2((info->z_size << 1) + i);
1316
1317         /*
1318          * Too big to be true, and overflowing left and right like mad ..
1319          */
1320         if ((info->z_full * align) < 1) {
1321                 if (adaptive == 0 && Z_IS_SINC(info)) {
1322                         adaptive = 1;
1323                         goto z_setup_adaptive_sinc;
1324                 }
1325                 return (E2BIG);
1326         }
1327
1328         /*
1329          * Increase full buffer size if its too small to reduce cyclic
1330          * buffer shifting in main conversion/feeder loop.
1331          */
1332         while (info->z_full < Z_RESERVOIR_MAX &&
1333             (info->z_full - (info->z_size << 1)) < Z_RESERVOIR)
1334                 info->z_full <<= 1;
1335
1336         /* Initialize buffer position. */
1337         info->z_mask = info->z_full - 1;
1338         info->z_start = z_prev(info, info->z_size << 1, 1);
1339         info->z_pos = z_next(info, info->z_start, 1);
1340
1341         /*
1342          * Allocate or reuse delay line buffer, whichever makes sense.
1343          */
1344         i = info->z_full * align;
1345         if (i < 1)
1346                 return (E2BIG);
1347
1348         if (info->z_delay == NULL || info->z_alloc < i ||
1349             i <= (info->z_alloc >> 1)) {
1350                 if (info->z_delay != NULL)
1351                         kfree(info->z_delay, M_DEVBUF);
1352                 info->z_delay = kmalloc(i, M_DEVBUF, M_WAITOK | M_ZERO);
1353                 if (info->z_delay == NULL)
1354                         return (ENOMEM);
1355                 info->z_alloc = i;
1356         }
1357
1358         /*
1359          * Zero out head of buffer to avoid pops and clicks.
1360          */
1361         memset(info->z_delay, sndbuf_zerodata(f->desc->out),
1362             info->z_pos * align);
1363
1364 #ifdef Z_DIAGNOSTIC
1365         /*
1366          * XXX Debuging mess !@#$%^
1367          */
1368 #define dumpz(x)        fprintf(stderr, "\t%12s = %10u : %-11d\n",      \
1369                             "z_"__STRING(x), (uint32_t)info->z_##x,     \
1370                             (int32_t)info->z_##x)
1371         fprintf(stderr, "\n%s():\n", __func__);
1372         fprintf(stderr, "\tchannels=%d, bps=%d, format=0x%08x, quality=%d\n",
1373             info->channels, info->bps, format, info->quality);
1374         fprintf(stderr, "\t%d (%d) -> %d (%d), ",
1375             info->src, info->rsrc, info->dst, info->rdst);
1376         fprintf(stderr, "[%d/%d]\n", info->z_gx, info->z_gy);
1377         fprintf(stderr, "\tminreq=%d, ", z_gy2gx(info, 1));
1378         if (adaptive != 0)
1379                 z_scale = Z_ONE;
1380         fprintf(stderr, "factor=0x%08x/0x%08x (%f)\n",
1381             z_scale, Z_ONE, (double)z_scale / Z_ONE);
1382         fprintf(stderr, "\tbase_length=%d, ", Z_SINC_BASE_LEN(info));
1383         fprintf(stderr, "adaptive=%s\n", (adaptive != 0) ? "YES" : "NO");
1384         dumpz(size);
1385         dumpz(alloc);
1386         if (info->z_alloc < 1024)
1387                 fprintf(stderr, "\t%15s%10d Bytes\n",
1388                     "", info->z_alloc);
1389         else if (info->z_alloc < (1024 << 10))
1390                 fprintf(stderr, "\t%15s%10d KBytes\n",
1391                     "", info->z_alloc >> 10);
1392         else if (info->z_alloc < (1024 << 20))
1393                 fprintf(stderr, "\t%15s%10d MBytes\n",
1394                     "", info->z_alloc >> 20);
1395         else
1396                 fprintf(stderr, "\t%15s%10d GBytes\n",
1397                     "", info->z_alloc >> 30);
1398         fprintf(stderr, "\t%12s   %10d (min output samples)\n",
1399             "",
1400             (int32_t)z_gx2gy(info, info->z_full - (info->z_size << 1)));
1401         fprintf(stderr, "\t%12s   %10d (min allocated output samples)\n",
1402             "",
1403             (int32_t)z_gx2gy(info, (info->z_alloc / align) -
1404             (info->z_size << 1)));
1405         fprintf(stderr, "\t%12s = %10d\n",
1406             "z_gy2gx()", (int32_t)z_gy2gx(info, 1));
1407         fprintf(stderr, "\t%12s = %10d -> z_gy2gx() -> %d\n",
1408             "Max", (int32_t)gy2gx_max, (int32_t)z_gy2gx(info, gy2gx_max));
1409         fprintf(stderr, "\t%12s = %10d\n",
1410             "z_gx2gy()", (int32_t)z_gx2gy(info, 1));
1411         fprintf(stderr, "\t%12s = %10d -> z_gx2gy() -> %d\n",
1412             "Max", (int32_t)gx2gy_max, (int32_t)z_gx2gy(info, gx2gy_max));
1413         dumpz(maxfeed);
1414         dumpz(full);
1415         dumpz(start);
1416         dumpz(pos);
1417         dumpz(scale);
1418         fprintf(stderr, "\t%12s   %10f\n", "",
1419             (double)info->z_scale / Z_ONE);
1420         dumpz(dx);
1421         fprintf(stderr, "\t%12s   %10f\n", "",
1422             (double)info->z_dx / info->z_dy);
1423         dumpz(dy);
1424         fprintf(stderr, "\t%12s   %10d (drift step)\n", "",
1425             info->z_dy >> Z_SHIFT);
1426         fprintf(stderr, "\t%12s   %10d (scaling differences)\n", "",
1427             (z_scale << Z_DRIFT_SHIFT) - info->z_dy);
1428         fprintf(stderr, "\t%12s = %u bytes\n",
1429             "intpcm32_t", sizeof(intpcm32_t));
1430         fprintf(stderr, "\t%12s = 0x%08x, smallest=%.16lf\n",
1431             "Z_ONE", Z_ONE, (double)1.0 / (double)Z_ONE);
1432 #endif
1433
1434         return (0);
1435 }
1436
1437 static int
1438 z_resampler_set(struct pcm_feeder *f, int what, int32_t value)
1439 {
1440         struct z_info *info;
1441         int32_t oquality;
1442
1443         info = f->data;
1444
1445         switch (what) {
1446         case Z_RATE_SRC:
1447                 if (value < feeder_rate_min || value > feeder_rate_max)
1448                         return (E2BIG);
1449                 if (value == info->rsrc)
1450                         return (0);
1451                 info->rsrc = value;
1452                 break;
1453         case Z_RATE_DST:
1454                 if (value < feeder_rate_min || value > feeder_rate_max)
1455                         return (E2BIG);
1456                 if (value == info->rdst)
1457                         return (0);
1458                 info->rdst = value;
1459                 break;
1460         case Z_RATE_QUALITY:
1461                 if (value < Z_QUALITY_MIN || value > Z_QUALITY_MAX)
1462                         return (EINVAL);
1463                 if (value == info->quality)
1464                         return (0);
1465                 /*
1466                  * If we failed to set the requested quality, restore
1467                  * the old one. We cannot afford leaving it broken since
1468                  * passive feeder chains like vchans never reinitialize
1469                  * itself.
1470                  */
1471                 oquality = info->quality;
1472                 info->quality = value;
1473                 if (z_resampler_setup(f) == 0)
1474                         return (0);
1475                 info->quality = oquality;
1476                 break;
1477         case Z_RATE_CHANNELS:
1478                 if (value < SND_CHN_MIN || value > SND_CHN_MAX)
1479                         return (EINVAL);
1480                 if (value == info->channels)
1481                         return (0);
1482                 info->channels = value;
1483                 break;
1484         default:
1485                 return (EINVAL);
1486                 break;
1487         }
1488
1489         return (z_resampler_setup(f));
1490 }
1491
1492 static int
1493 z_resampler_get(struct pcm_feeder *f, int what)
1494 {
1495         struct z_info *info;
1496
1497         info = f->data;
1498
1499         switch (what) {
1500         case Z_RATE_SRC:
1501                 return (info->rsrc);
1502                 break;
1503         case Z_RATE_DST:
1504                 return (info->rdst);
1505                 break;
1506         case Z_RATE_QUALITY:
1507                 return (info->quality);
1508                 break;
1509         case Z_RATE_CHANNELS:
1510                 return (info->channels);
1511                 break;
1512         default:
1513                 break;
1514         }
1515
1516         return (-1);
1517 }
1518
1519 static int
1520 z_resampler_init(struct pcm_feeder *f)
1521 {
1522         struct z_info *info;
1523         int ret;
1524
1525         if (f->desc->in != f->desc->out)
1526                 return (EINVAL);
1527
1528         info = kmalloc(sizeof(*info), M_DEVBUF, M_WAITOK | M_ZERO);
1529         if (info == NULL)
1530                 return (ENOMEM);
1531
1532         info->rsrc = Z_RATE_DEFAULT;
1533         info->rdst = Z_RATE_DEFAULT;
1534         info->quality = feeder_rate_quality;
1535         info->channels = AFMT_CHANNEL(f->desc->in);
1536
1537         f->data = info;
1538
1539         ret = z_resampler_setup(f);
1540         if (ret != 0) {
1541                 if (info->z_pcoeff != NULL)
1542                         kfree(info->z_pcoeff, M_DEVBUF);
1543                 if (info->z_delay != NULL)
1544                         kfree(info->z_delay, M_DEVBUF);
1545                 kfree(info, M_DEVBUF);
1546                 f->data = NULL;
1547         }
1548
1549         return (ret);
1550 }
1551
1552 static int
1553 z_resampler_free(struct pcm_feeder *f)
1554 {
1555         struct z_info *info;
1556
1557         info = f->data;
1558         if (info != NULL) {
1559                 if (info->z_pcoeff != NULL)
1560                         kfree(info->z_pcoeff, M_DEVBUF);
1561                 if (info->z_delay != NULL)
1562                         kfree(info->z_delay, M_DEVBUF);
1563                 kfree(info, M_DEVBUF);
1564         }
1565
1566         f->data = NULL;
1567
1568         return (0);
1569 }
1570
1571 static uint32_t
1572 z_resampler_feed_internal(struct pcm_feeder *f, struct pcm_channel *c,
1573     uint8_t *b, uint32_t count, void *source)
1574 {
1575         struct z_info *info;
1576         int32_t alphadrift, startdrift, reqout, ocount, reqin, align;
1577         int32_t fetch, fetched, start, cp;
1578         uint8_t *dst;
1579
1580         info = f->data;
1581         if (info->z_resample == NULL)
1582                 return (z_feed(f->source, c, b, count, source));
1583
1584         /*
1585          * Calculate sample size alignment and amount of sample output.
1586          * We will do everything in sample domain, but at the end we
1587          * will jump back to byte domain.
1588          */
1589         align = info->channels * info->bps;
1590         ocount = SND_FXDIV(count, align);
1591         if (ocount == 0)
1592                 return (0);
1593
1594         /*
1595          * Calculate amount of input samples that is needed to generate
1596          * exact amount of output.
1597          */
1598         reqin = z_gy2gx(info, ocount) - z_fetched(info);
1599
1600 #ifdef Z_USE_ALPHADRIFT
1601         startdrift = info->z_startdrift;
1602         alphadrift = info->z_alphadrift;
1603 #else
1604         startdrift = _Z_GY2GX(info, 0, 1);
1605         alphadrift = z_drift(info, startdrift, 1);
1606 #endif
1607
1608         dst = b;
1609
1610         do {
1611                 if (reqin != 0) {
1612                         fetch = z_min(z_free(info), reqin);
1613                         if (fetch == 0) {
1614                                 /*
1615                                  * No more free spaces, so wind enough
1616                                  * samples back to the head of delay line
1617                                  * in byte domain.
1618                                  */
1619                                 fetched = z_fetched(info);
1620                                 start = z_prev(info, info->z_start,
1621                                     (info->z_size << 1) - 1);
1622                                 cp = (info->z_size << 1) + fetched;
1623                                 z_copy(info->z_delay + (start * align),
1624                                     info->z_delay, cp * align);
1625                                 info->z_start =
1626                                     z_prev(info, info->z_size << 1, 1);
1627                                 info->z_pos =
1628                                     z_next(info, info->z_start, fetched + 1);
1629                                 fetch = z_min(z_free(info), reqin);
1630 #ifdef Z_DIAGNOSTIC
1631                                 if (1) {
1632                                         static uint32_t kk = 0;
1633                                         fprintf(stderr,
1634                                             "Buffer Move: "
1635                                             "start=%d fetched=%d cp=%d "
1636                                             "cycle=%u [%u]\r",
1637                                             start, fetched, cp, info->z_cycle,
1638                                             ++kk);
1639                                 }
1640                                 info->z_cycle = 0;
1641 #endif
1642                         }
1643                         if (fetch != 0) {
1644                                 /*
1645                                  * Fetch in byte domain and jump back
1646                                  * to sample domain.
1647                                  */
1648                                 fetched = SND_FXDIV(z_feed(f->source, c,
1649                                     info->z_delay + (info->z_pos * align),
1650                                     fetch * align, source), align);
1651                                 /*
1652                                  * Prepare to convert fetched buffer,
1653                                  * or mark us done if we cannot fulfill
1654                                  * the request.
1655                                  */
1656                                 reqin -= fetched;
1657                                 info->z_pos += fetched;
1658                                 if (fetched != fetch)
1659                                         reqin = 0;
1660                         }
1661                 }
1662
1663                 reqout = z_min(z_gx2gy(info, z_fetched(info)), ocount);
1664                 if (reqout != 0) {
1665                         ocount -= reqout;
1666
1667                         /*
1668                          * Drift.. drift.. drift..
1669                          *
1670                          * Notice that there are 2 methods of doing the drift
1671                          * operations: The former is much cleaner (in a sense
1672                          * of mathematical readings of my eyes), but slower
1673                          * due to integer division in z_gy2gx(). Nevertheless,
1674                          * both should give the same exact accurate drifting
1675                          * results, so the later is favourable.
1676                          */
1677                         do {
1678                                 info->z_resample(info, dst);
1679 #if 0
1680                                 startdrift = z_gy2gx(info, 1);
1681                                 alphadrift = z_drift(info, startdrift, 1);
1682                                 info->z_start += startdrift;
1683                                 info->z_alpha += alphadrift;
1684 #else
1685                                 info->z_alpha += alphadrift;
1686                                 if (info->z_alpha < info->z_gy)
1687                                         info->z_start += startdrift;
1688                                 else {
1689                                         info->z_start += startdrift - 1;
1690                                         info->z_alpha -= info->z_gy;
1691                                 }
1692 #endif
1693                                 dst += align;
1694 #ifdef Z_DIAGNOSTIC
1695                                 info->z_cycle++;
1696 #endif
1697                         } while (--reqout != 0);
1698                 }
1699         } while (reqin != 0 && ocount != 0);
1700
1701         /*
1702          * Back to byte domain..
1703          */
1704         return (dst - b);
1705 }
1706
1707 static int
1708 z_resampler_feed(struct pcm_feeder *f, struct pcm_channel *c, uint8_t *b,
1709     uint32_t count, void *source)
1710 {
1711         uint32_t feed, maxfeed, left;
1712
1713         /*
1714          * Split count to smaller chunks to avoid possible 32bit overflow.
1715          */
1716         maxfeed = ((struct z_info *)(f->data))->z_maxfeed;
1717         left = count;
1718
1719         do {
1720                 feed = z_resampler_feed_internal(f, c, b,
1721                     z_min(maxfeed, left), source);
1722                 b += feed;
1723                 left -= feed;
1724         } while (left != 0 && feed != 0);
1725
1726         return (count - left);
1727 }
1728
1729 static struct pcm_feederdesc feeder_rate_desc[] = {
1730         { FEEDER_RATE, 0, 0, 0, 0 },
1731         { 0, 0, 0, 0, 0 },
1732 };
1733
1734 static kobj_method_t feeder_rate_methods[] = {
1735         KOBJMETHOD(feeder_init,         z_resampler_init),
1736         KOBJMETHOD(feeder_free,         z_resampler_free),
1737         KOBJMETHOD(feeder_set,          z_resampler_set),
1738         KOBJMETHOD(feeder_get,          z_resampler_get),
1739         KOBJMETHOD(feeder_feed,         z_resampler_feed),
1740         KOBJMETHOD_END
1741 };
1742
1743 FEEDER_DECLARE(feeder_rate, NULL);