Ansify some remaining function definitions in the kernel.
[dragonfly.git] / sys / dev / atm / hea / eni_receive.c
1 /*
2  *
3  * ===================================
4  * HARP  |  Host ATM Research Platform
5  * ===================================
6  *
7  *
8  * This Host ATM Research Platform ("HARP") file (the "Software") is
9  * made available by Network Computing Services, Inc. ("NetworkCS")
10  * "AS IS".  NetworkCS does not provide maintenance, improvements or
11  * support of any kind.
12  *
13  * NETWORKCS MAKES NO WARRANTIES OR REPRESENTATIONS, EXPRESS OR IMPLIED,
14  * INCLUDING, BUT NOT LIMITED TO, IMPLIED WARRANTIES OF MERCHANTABILITY
15  * AND FITNESS FOR A PARTICULAR PURPOSE, AS TO ANY ELEMENT OF THE
16  * SOFTWARE OR ANY SUPPORT PROVIDED IN CONNECTION WITH THIS SOFTWARE.
17  * In no event shall NetworkCS be responsible for any damages, including
18  * but not limited to consequential damages, arising from or relating to
19  * any use of the Software or related support.
20  *
21  * Copyright 1994-1998 Network Computing Services, Inc.
22  *
23  * Copies of this Software may be made, however, the above copyright
24  * notice must be reproduced on all copies.
25  *
26  *      @(#) $FreeBSD: src/sys/dev/hea/eni_receive.c,v 1.5 1999/08/28 00:41:45 peter Exp $
27  *      @(#) $DragonFly: src/sys/dev/atm/hea/eni_receive.c,v 1.10 2008/03/01 22:03:13 swildner Exp $
28  */
29
30 /*
31  * Efficient ENI Adapter Support
32  * -----------------------------
33  *
34  * Receive management
35  *
36  */
37
38 #include <netproto/atm/kern_include.h>
39
40 #include "eni_stats.h"
41 #include "eni.h"
42 #include "eni_var.h"
43
44 static void     eni_recv_stack (void *, KBuffer *);
45
46 #ifdef  DIAGNOSTIC
47 extern int      eni_pdu_print;
48 #endif
49
50 /*
51  * Procedure to remove VCs from the Service List and generate DMA
52  * requests to move the associated PDUs into host memory. As PDUs
53  * are completed in adapter memory, the adapter examines the IN_SERVICE
54  * bit for the VC in the VC table. If this bit is not set, the adapter
55  * will place the VC number at the end of the service list queue, set
56  * the IN_SERVICE bit in the VC table, and interrupt the host. The host
57  * will remove VCs from the service list, clear the IN_SERVICE bit in
58  * the VC table, and create a DMA list to move the PDU into host buffers.
59  *
60  * Arguments:
61  *      eup             pointer to per unit structure
62  *
63  * Returns:
64  *      none
65  *
66  */
67 void
68 eni_do_service(Eni_unit *eup)
69 {
70         int             vcc;
71         Eni_vcc         *evp;
72         u_long          servwrite;
73         VCI_Table       *vct;
74         u_long          rdptr;
75         u_long          *rxp;
76         KBuffer         *m;
77         u_long          dma[TEMP_DMA_SIZE];
78         u_long          i, j;
79         u_long          dma_rd, dma_wr;
80         u_long          dma_avail;
81         int             pdulen;
82         int             mask;
83         u_long          *upp;
84
85         /*
86          * Where is the adapter currently inserting entries?
87          */
88         servwrite = eup->eu_midway[MIDWAY_SVCWR] & SVC_SIZE_MASK;
89         /*
90          * As long as we're not caught up with the adapter, keep
91          * removing VCs from the service list.
92          */
93         while ( servwrite != eup->eu_servread ) {
94                 int     vci_hdr;
95                 u_long  descr;
96
97                 /*
98                  * Get VC number and find VC table entry.
99                  */
100                 vcc = eup->eu_svclist[eup->eu_servread];
101                 vct = &eup->eu_vcitbl[vcc];
102                 vci_hdr = vct->vci_control;     /* Current status */
103
104                 /*
105                  * Check that this VCC still needs servicing. We
106                  * might have closed this VCC down in between
107                  * the adapter setting the flag and our checking
108                  * the flag. Also check that we haven't placed the
109                  * VCC into TRASH mode.
110                  */
111                 if ( ( vci_hdr & VCI_IN_SERVICE ) == 0 ||
112                     ( (vci_hdr & ~VCI_MODE_MASK) ==
113                         (VCI_MODE_TRASH << VCI_MODE_SHIFT) ) )
114                             goto next_vcc;
115
116                 /*
117                  * Find the size of this VCs buffer
118                  */
119                 mask = (vci_hdr >> VCI_SIZE_SHIFT) & VCI_SIZE_MASK;
120                 mask = 1 << (ENI_LOC_PREDIV + mask);
121                 /* Turn byte count into word count */
122                 mask >>= 2;
123                 /*
124                  * Find the start of the adapter buffer for this VC.
125                  */
126                 rxp = (u_long *)
127                     ((int)(((vci_hdr >> VCI_LOC_SHIFT ) & VCI_LOC_MASK)
128                         << ENI_LOC_PREDIV) + (int)eup->eu_ram);
129                 /*
130                  * Locate incoming VCC for this PDU and find where we
131                  * should next read from.
132                  */
133                 evp = (Eni_vcc *) atm_dev_vcc_find ( (Cmn_unit *)eup,
134                     0, vcc, VCC_IN );
135                 if ( evp == (Eni_vcc *)NULL )
136                         goto next_vcc;          /* VCI no longer active */
137                 rdptr = evp->ev_rxpos;
138                 /*
139                  * Find out where the adapter is currently reassembling.
140                  * The PDU which starts at descr is not yet complete so we
141                  * must stop there.
142                  */
143                 descr = ( vct->vci_descr >> 16 ) & 0x7FFF;
144                 /*
145                  * As long as we haven't processed all the completed PDUs on
146                  * this VC, keep going...
147                  */
148                 while ( rdptr != descr )
149                 {
150                     int         n_cells;
151                     int         pdu_descr;
152                     int         aal5;
153
154                     /*
155                      * Ensure that the following are reset for every new
156                      * PDU.
157                      */
158                     upp = NULL;
159                     m = NULL;
160
161                     /*
162                      * Fisrt build a DMA with JK to skip the descriptor word.
163                      * We must always skip the descriptor even if it turns out
164                      * that there isn't any PDU here.
165                      */
166                     j = 0;
167                     dma[j++] = (((rdptr + 1) & (mask-1)) << DMA_COUNT_SHIFT ) |
168                         ( vcc << DMA_VCC_SHIFT ) | DMA_JK;
169                     dma[j++] = 0;
170
171                     /*
172                      * We'll use some of the values below for skipping
173                      * bad PDUs or counting statistics so compute them
174                      * now.
175                      */
176
177                     /*
178                      * Grab a copy of the descriptor word
179                      */
180                     pdu_descr = rxp[rdptr];
181
182                     /*
183                      * Strip out cell count from descriptor word.
184                      * At this point, we still don't know if there
185                      * is any real data until after we check for
186                      * TRASH mode.
187                      */
188                     n_cells = pdu_descr & DESCR_CELL_COUNT;
189
190                     /*
191                      * Is this an AAL5 PDU? Check MODE in vci_hdr.
192                      */
193                     aal5 = ( ( vci_hdr & ~VCI_MODE_MASK ) ==
194                         VCI_MODE_AAL5 << VCI_MODE_SHIFT );
195
196                     /*
197                      * Now check to see if we're trashing on this vcc.
198                      * If so, there is no data with this VC and the
199                      * next word after the current descriptor is the
200                      * descriptor for the next PDU.
201                      */
202                     if ( ( pdu_descr & DESCR_TRASH_BIT ) != 0 ) {
203                         if ( aal5 )
204                                 /*
205                                  * Count as number of AAL5 cells dropped
206                                  */
207                                 eup->eu_stats.eni_st_aal5.aal5_drops += n_cells;
208                         else
209                                 /*
210                                  * Count as number of AAL0 cells dropped
211                                  */
212                                 eup->eu_stats.eni_st_aal0.aal0_drops += n_cells;
213                         eup->eu_pif.pif_ierrors++;
214                         /*
215                          * When cells have been trashed, all we have in the
216                          * buffer is a descriptor word. There are no data
217                          * words. Set the number of cells to zero so that
218                          * we correctly skip to the next word which will
219                          * be the descriptor for the next PDU.
220                          */
221                         n_cells = 0;
222                         /*
223                          * Go issue the DMA to skip this descriptor word.
224                          */
225                         goto send_dma;
226                     }
227
228                     /*
229                      * Data length: number of cells * cell size
230                      */
231                     pdulen = n_cells * BYTES_PER_CELL;
232
233                     /*
234                      * If this is an AAL5 PDU, then we need to check
235                      * for the presence of any CRC errors. If there
236                      * is one or more CRC errors, then we are going to
237                      * drop this PDU.
238                      */
239                     if ( aal5 && ( pdu_descr & DESCR_CRC_ERR ) ) {
240                         /*
241                          * Count the stat
242                          */
243                         eup->eu_pif.pif_ierrors++;
244                         eup->eu_stats.eni_st_aal5.aal5_pdu_crc++;
245                         if ( evp->ev_connvc->cvc_vcc )
246                                 evp->ev_connvc->cvc_vcc->vc_ierrors++;
247                         /*
248                          * Build a DMA entry to skip the rest of this
249                          * PDU.
250                          */
251                         dma[j++] =
252                             (((rdptr + n_cells*WORDS_PER_CELL + 1)
253                                 & (mask-1)) << DMA_COUNT_SHIFT ) |
254                                     (vcc << DMA_VCC_SHIFT ) | DMA_JK;
255                         dma[j++] = 0;
256                         /*
257                          * All done with this PDU. Get a buffer to save some
258                          * data for reclamation services.
259                          */
260                         KB_ALLOCPKT ( m, ENI_SMALL_BSIZE, KB_F_NOWAIT,
261                             KB_T_DATA );
262                         if ( m ) {
263                                 u_long  *up;
264
265                                 KB_DATASTART ( m, up, u_long * );
266                                 /*
267                                  * Indicate no PDU
268                                  */
269                                 KB_PLENSET ( m, 0 );
270                                 /*
271                                  * Set buffer length - only driver overhead
272                                  */
273                                 KB_LEN ( m ) = 3 * sizeof ( u_long );
274                                 /*
275                                  * Insert vcc, space for DMA pointers,
276                                  * and pdulen
277                                  */
278                                 *up++ = vcc;
279                                 upp = up;       /* Remember location */
280                                 up++;           /* And skip it */
281                                                 /* - to be filled later */
282                                 *up = pdulen;   /* Actual PDU length if it */
283                                                 /* were valid */
284                         } else {
285                                 /*
286                                  * We've a real problem here as now we can't
287                                  * reclaim/advance resources/safety pointers.
288                                  */
289                                 eup->eu_stats.eni_st_drv.drv_rv_norsc++;
290 #ifdef  DO_LOG
291                                 log ( LOG_ERR,
292     "eni_do_service: No drain buffers available. Receiver about to lock.\n" );
293 #endif
294                         }
295                         goto send_dma;
296                     }
297
298                     /*
299                      * Do we need to strip the AAL layer? Yes if this
300                      * is an AAL5 PDU.
301                      */
302                     if ( aal5 ) {
303                         /*
304                          * Grab the CS-PDU length. Find the address of the
305                          * last word, back up one word to skip CRC, and
306                          * then mask the whole thing to handle circular wraps.
307                          */
308                         pdulen = rxp[(rdptr + n_cells*WORDS_PER_CELL - 1)
309                             & (mask-1)]
310                                 & 0xFFFF;
311                     }
312
313                     /*
314                      * We now have a valid PDU of some length. Build
315                      * the necessary DMA list to move it into host
316                      * memory.
317                      */
318
319                     /*
320                      * Get an initial buffer.
321                      */
322                     KB_ALLOCPKT ( m, ENI_SMALL_BSIZE, KB_F_NOWAIT, KB_T_DATA );
323                     /*
324                      * Do we have a valid buffer?
325                      */
326                     if ( m != (KBuffer *)NULL )
327                     {
328                         int     len;
329                         u_long  *up;
330                         KBuffer *m0;
331         
332                         KB_DATASTART ( m, up, u_long * );
333                         /*
334                          * Fill in pdulen in PKTHDR structure (for IP).
335                          */
336                         KB_PLENSET ( m, pdulen );
337                         /*
338                          * We're going to save the VCI nuber, the start
339                          * and stop DMA pointers, and the PDU length at
340                          * the head of the buffer. We'll pull this out
341                          * later after the DMA has completed.
342                          *
343                          * Insert VCI number as first word in first buffer,
344                          * remeber where we want to store the start/stop
345                          * pointers, and store the PDU length.
346                          */
347                         *up++ = vcc;    /* PDU's VCC */
348                         upp = up;       /* Remember where we are */
349                         up++;           /* To stuff start/stop pointers in */
350                         *up++ = pdulen; /* PDU's length */
351                         /*
352                          * Leave some extra room in case a higher protocol
353                          * (IP) wants to do a pullup. Maybe we can keep
354                          * someone from having to allocate another buffer
355                          * a do a larger memory copy.
356                          */
357                         len = MIN ( ENI_SMALL_BSIZE, pdulen );
358                         (void) eni_set_dma ( eup, 1, dma, TEMP_DMA_SIZE, &j,
359                                 vcc, (u_long)up, len );
360                         /*
361                          * Adjust length of remaining data in PDU
362                          */
363                         pdulen -= len;
364                         /*
365                          * Set buffer length, including our overhead
366                          */
367                         KB_LEN ( m ) = len + 3 * sizeof ( u_long );
368                         /*
369                          * Finish by moving anything which won't fit in
370                          * first buffer
371                          */
372                         m0 = m;
373                         while ( pdulen ) {
374                                 KBuffer *m1;
375                                 u_long  data_addr;
376         
377                                 /*
378                                  * Get another buffer
379                                  */
380                                 KB_ALLOCEXT ( m1, ENI_LARGE_BSIZE, KB_F_NOWAIT,
381                                         KB_T_DATA );
382         
383                                 /*
384                                  * If we succeeded...
385                                  */
386                                 if ( m1 ) {
387                                     /*
388                                      * Figure out how much we can move into
389                                      * this buffer.
390                                      */
391                                     len = MIN ( ENI_LARGE_BSIZE, pdulen );
392                                     /*
393                                      * Setup DMA list for this buffer
394                                      */
395                                     KB_DATASTART ( m1, data_addr, u_long );
396                                     (void) eni_set_dma
397                                         ( eup, 1, dma, TEMP_DMA_SIZE, &j, vcc,
398                                             data_addr, len );
399                                     /*
400                                      * Adjust remaining length
401                                      */
402                                     pdulen -= len;
403                                     /*
404                                      * Set buffer length
405                                      */
406                                     KB_LEN ( m1 ) = len;
407                                     /*
408                                      * Link new buffer onto end and advance
409                                      * pointer
410                                      */
411                                     KB_NEXT ( m0 ) = m1;
412                                     m0 = m1;
413                                 } else {
414                                     /*
415                                      * Either we were unable to grab another
416                                      * buffer or there are no large buffers
417                                      * available. We know that the first
418                                      * buffer is valid, so drop everything
419                                      * else, build a JK DMA to skip/drop this
420                                      * PDU, set the pointers to reclaim
421                                      * resources/advance pointers, and
422                                      * finish this PDU now.
423                                      */
424                                     if ( KB_NEXT ( m ) )
425                                         KB_FREEALL ( KB_NEXT ( m ) );
426                                     eup->eu_pif.pif_ierrors++;
427                                     j = 2;
428                                     dma[j++] =
429                                         (((rdptr + n_cells*WORDS_PER_CELL + 1)
430                                             & (mask-1)) << DMA_COUNT_SHIFT ) |
431                                                 (vcc << DMA_VCC_SHIFT ) |
432                                                     DMA_JK;
433                                     dma[j++] = 0;
434                                     /*
435                                      * Reset PDU length to zero
436                                      */
437                                     KB_PLENSET ( m, 0 );
438                                     /*
439                                      * Count some statistics
440                                      */
441                                     /*
442                                      * Count this as dropped cells
443                                      */
444                                     if ( aal5 ) {
445                                         eup->eu_stats.eni_st_aal5.aal5_drops +=
446                                             n_cells;
447                                         eup->eu_stats.eni_st_aal5.aal5_pdu_drops++;
448                                     } else
449                                         eup->eu_stats.eni_st_aal0.aal0_drops +=
450                                             n_cells;
451                                     /*
452                                      * Drop it
453                                      */
454                                     goto send_dma;
455                                 }
456                         }
457                         /*
458                          * If necessary, skip AAL layer
459                          */
460                         if ( aal5 ) {
461                                 dma[j++] =
462                                   (((rdptr + n_cells*WORDS_PER_CELL + 1)
463                                         & (mask-1)) << DMA_COUNT_SHIFT)
464                                             | (vcc << DMA_VCC_SHIFT) | DMA_JK;
465                                 dma[j++] = 0;
466                         }
467                     } else {
468                         /*
469                          * We failed to get an initial buffer. Since we
470                          * haven't changed anything for this PDU yet and the
471                          * PDU is still valid, exit now and try to service it
472                          * next time around. We're not very likely to get
473                          * another buffer right now anyways.
474                          */
475                         eup->eu_stats.eni_st_drv.drv_rv_nobufs++;
476 #ifdef  DO_LOG
477                         log ( LOG_ERR,
478 "eni_do_service: No buffers available. Exiting without servicing service list.\n" );
479 #endif
480                         /*
481                          * Clear the IN_SERVICE indicator for this VCC
482                          */
483                         vct->vci_control &= ~VCI_IN_SERVICE;
484                         return;
485                     }
486
487 send_dma:
488                     /*
489                      * Set the end bit on the last DMA for this PDU
490                      */
491                     dma[j-2] |= DMA_END_BIT;
492
493                     /*
494                      * Where are the current DMA pointers
495                      */
496                     dma_rd = eup->eu_midway[MIDWAY_RX_RD];
497                     dma_wr = eup->eu_midway[MIDWAY_RX_WR];
498
499                     /*
500                      * Check how much space is available
501                      */
502                     if ( dma_rd == dma_wr )
503                         dma_avail = DMA_LIST_SIZE;
504                     else
505                         dma_avail = ( dma_rd + DMA_LIST_SIZE - dma_wr )
506                             & (DMA_LIST_SIZE-1);
507
508                     /*
509                      * Check for queue full or wrap past write okay pointer
510                      */
511                     if ( dma_avail < j  ||
512                         ( dma_wr + j > eup->eu_rxdmawr + DMA_LIST_SIZE ) ) {
513                         /*
514                          * There's no room in the DMA list to insert
515                          * this request. Since we haven't changed anything
516                          * yet and the PDU is good, exit now and service
517                          * it next time around. What we really need to do
518                          * is wait for the RX list to drain and that won't
519                          * happen if we keep trying to process PDUs here.
520                          */
521                         eup->eu_stats.eni_st_drv.drv_rv_nodma++;
522 #ifdef  DO_LOG
523                         log ( LOG_ERR,
524 "eni_do_service: No room in receive DMA list. Postponing service request.\n" );
525 #endif
526                         /*
527                          * Free the local buffer chain
528                          */
529                         KB_FREEALL ( m );
530                         /*
531                          * Clear the IN_SERVICE indicator for this VCC.
532                          */
533                         vct->vci_control &= ~VCI_IN_SERVICE;
534                         return; 
535                     }
536
537                     /*
538                      * If we have a buffer chain, save the starting
539                      * dma_list location.
540                      */
541                     if ( upp ) {
542                         *upp = dma_wr << 16;
543                     }
544
545                     /*
546                      * Stuff the DMA list
547                      */
548                     j >>= 1;
549                     for ( i = 0; i < j; i++ ) {
550                         eup->eu_rxdma[dma_wr*2] = dma[i*2];
551                         eup->eu_rxdma[dma_wr*2+1] = dma[i*2+1];
552                         dma_wr = (dma_wr+1) & (DMA_LIST_SIZE-1);
553                     }
554                     /*
555                      * If we have a buffer chain, save the location of
556                      * the ending dma_list location and queue the chain
557                      * so that we can recover the resources later.
558                      */
559                     if ( upp ) {
560                         *upp |= dma_wr;
561                         /*
562                          * Place buffer on receive queue waiting for RX_DMA
563                          */
564                         if ( IF_QFULL ( &eup->eu_rxqueue ) ) {
565                             /*
566                              * We haven't done anything we can't back out
567                              * of. Drop request and service it next time.
568                              * We've inserted the DMA list but it's not
569                              * valid until we advance the RX_WR pointer,
570                              * thus it's okay to bail here...
571                              */
572                             eup->eu_stats.eni_st_drv.drv_rv_rxq++;
573 #ifdef  DO_LOG
574                             log ( LOG_ERR,
575         "eni_do_service: RX drain queue full. Postponing servicing.\n" );
576 #endif
577                             KB_FREEALL ( m );
578                             /*
579                              * Clear the IN_SERVICE indicator for this VCC.
580                              */
581                             vct->vci_control &= ~VCI_IN_SERVICE;
582                             return;
583                         } else { 
584                             IF_ENQUEUE ( &eup->eu_rxqueue, m );
585                             /*
586                              * Advance the RX_WR pointer to cause
587                              * the adapter to work on this DMA list.
588                              */
589                             eup->eu_midway[MIDWAY_RX_WR] = dma_wr;
590                         }
591                     }
592                     /*
593                      * Advance our notion of where the next PDU
594                      * should start.
595                      */
596                     rdptr = (rdptr + n_cells*WORDS_PER_CELL + 1)
597                         & (mask-1);
598                     evp->ev_rxpos = rdptr;
599
600                     /*
601                      * Increment cells/pdu received stats.
602                      */
603                     eup->eu_stats.eni_st_atm.atm_rcvd += n_cells;
604                     if ( aal5 ) {
605                         eup->eu_stats.eni_st_aal5.aal5_rcvd += n_cells;
606                         eup->eu_stats.eni_st_aal5.aal5_pdu_rcvd++;
607                     } else {
608                         eup->eu_stats.eni_st_aal0.aal0_rcvd += n_cells;
609                     }
610
611                     /*
612                      * Continue processing PDUs on this same VCI
613                      */
614                 }
615
616 next_vcc:
617                 /*
618                  * Advance to next entry in the service_list.
619                  */
620                 eup->eu_servread = (eup->eu_servread + 1) & SVC_SIZE_MASK;
621
622                 /*
623                  * And clear the IN_SERVICE indicator for this VCC.
624                  */
625                 vct->vci_control &= ~VCI_IN_SERVICE;
626         }
627         return;
628 }
629
630 /*
631  * Drain Receive queue
632  *
633  * As we build DMA lists to move PDUs from adapter buffers into host
634  * buffers, we place the request on a private ifqueue so that we can
635  * free any resources AFTER we know they've been successfully DMAed.
636  * As part of the service processing, we record the PDUs start and stop
637  * entries in the DMA list, and prevent wrapping. When we pull the top
638  * entry off, we simply check that the current DMA location is outside
639  * this PDU and if so, it's okay to free things.
640  *
641  * Arguments:
642  *      eup             pointer to device unit structure
643  *
644  * Returns:
645  *      none
646  *
647  */
648 void
649 eni_recv_drain(Eni_unit *eup)
650 {
651         KBuffer         *m;
652         Eni_vcc         *evp;
653         struct vccb     *vcp;
654         u_long          vcc;
655         u_long          DMA_Rdptr;
656         u_long          dma_wrp;
657         u_long          start, stop;
658
659         crit_enter();
660         /* Pop first buffer */
661         IF_DEQUEUE ( &eup->eu_rxqueue, m );
662         while ( m ) {
663                 u_long  *up;
664                 u_long  pdulen;
665
666                 KB_DATASTART ( m, up, u_long * );
667
668                 /*
669                  * Grab the VCI number
670                  */
671                 vcc = *up++;
672
673                 /*
674                  * Check to see if we can process this buffer yet.
675                  */
676                 /* Get current DMA_Rdptr */
677                 DMA_Rdptr = eup->eu_midway[MIDWAY_RX_RD];
678                 /* Boundaries for first buffer */
679                 dma_wrp = *up++;
680                 start = dma_wrp >> 16;
681                 stop = dma_wrp & 0xffff;
682                 /*
683                  * Start should not equal stop because that would
684                  * mean we tried inserting a NULL DMA list.
685                  */
686                 if ( start > stop ) {           /* We wrapped */
687                         if ( !(DMA_Rdptr >= stop && DMA_Rdptr < start) ) {
688                                 IF_PREPEND ( &eup->eu_rxqueue, m );
689                                 goto finish;
690                         }
691                 } else {
692                         if ( DMA_Rdptr < stop && DMA_Rdptr >= start ) {
693                                 IF_PREPEND ( &eup->eu_rxqueue, m );
694                                 goto finish;
695                         }
696                 }
697                 /*
698                  * Adapter is finished with this buffer, we can
699                  * continue processing it now.
700                  */
701
702                 /*
703                  * Locate incoming VCC for this PDU
704                  */
705                 evp = (Eni_vcc *) atm_dev_vcc_find ( (Cmn_unit *)eup,
706                     0, vcc, VCC_IN );
707
708                 if ( evp == NULL ) {
709                         eup->eu_stats.eni_st_drv.drv_rv_novcc++;
710                         KB_FREEALL ( m );
711                         goto next_buffer;
712                 }
713
714 #ifdef  DIAGNOSTIC
715                 if ( eni_pdu_print )
716                     atm_dev_pdu_print ( (Cmn_unit *)eup, (Cmn_vcc *)evp, m,
717                         "eni_stack_drain" );
718 #endif
719
720                 /*
721                  * Grab theoretical PDU length
722                  */
723                 pdulen = *up++;
724
725                 /*
726                  * Quick, count the PDU
727                  */
728                 eup->eu_pif.pif_ipdus++;
729                 eup->eu_pif.pif_ibytes += pdulen;
730                 if ( evp ) {
731                     vcp = evp->ev_connvc->cvc_vcc;
732                     if ( vcp ) {
733                         vcp->vc_ipdus++;
734                         vcp->vc_ibytes += pdulen;
735                         if ( vcp->vc_nif ) {
736                             vcp->vc_nif->nif_ibytes += pdulen;
737                             vcp->vc_nif->nif_if.if_ipackets++;
738                             vcp->vc_nif->nif_if.if_ibytes += pdulen;
739                         }
740                     }
741                 }
742
743                 /*
744                  * Advance DMA write allowable pointer
745                  */
746                 eup->eu_rxdmawr = stop;
747
748                 /*
749                  * Get packet PDU length
750                  */
751                 KB_PLENGET ( m, pdulen );
752
753                 /*
754                  * Only try queueing this if there is data
755                  * to be handed up to the next layer. Errors
756                  * such as CRC and VC trashing will get us this
757                  * far to advance pointers, etc., but the PDU
758                  * length will be zero.
759                  */
760                 if ( pdulen ) {
761                         /*
762                          * We saved three words back in eni_do_service()
763                          * to use for callback. Since the core only
764                          * expects two words, skip over the first one.
765                          * Then, reset up pointer to start of buffer data
766                          * area and write the callback info.
767                          */
768                         KB_HEADADJ ( m, -sizeof(u_long) );
769                         KB_DATASTART ( m, up, u_long * );
770                         *((int *)up) = (int)eni_recv_stack;
771                         up++;
772                         *((int *)up) = (int)evp;
773                         /*
774                          * Schedule callback
775                          */
776                         if (netisr_queue(NETISR_ATM, m)) {
777                                 eup->eu_stats.eni_st_drv.drv_rv_intrq++;
778                                 eup->eu_pif.pif_ierrors++;
779 #ifdef  DO_LOG
780                                 log ( LOG_ERR,
781 "eni_receive_drain: ATM_INTRQ is full. Unable to pass up stack.\n" );
782 #endif
783                                 KB_FREEALL ( m );
784                         }
785                 } else {
786                         /*
787                          * Free zero-length buffer
788                          */
789                         KB_FREEALL(m);
790                 }
791
792 next_buffer:
793                 /*
794                  * Look for next buffer
795                  */
796                 IF_DEQUEUE ( &eup->eu_rxqueue, m );
797         }
798 finish:
799         crit_exit();
800 }
801
802 /*
803  * Pass incoming PDU up Stack
804  *
805  * This function is called via the core ATM interrupt queue callback
806  * set in eni_recv_drain(). It will pass the supplied incoming
807  * PDU up the incoming VCC's stack.
808  *
809  * Arguments:
810  *      tok             token to identify stack instantiation
811  *      m               pointer to incoming PDU buffer chain
812  *
813  * Returns:
814  *      none
815  */
816 static void
817 eni_recv_stack(void *tok, KBuffer *m)
818 {
819         Eni_vcc         *evp = (Eni_vcc *)tok;
820         int             err;
821
822         /*
823          * This should never happen now but if it does and we don't stop it,
824          * we end up panic'ing in netatm when trying to pull a function
825          * pointer and token value out of a buffer with address zero.
826          */
827         if ( !m ) {
828 #ifdef  DO_LOG
829                 log ( LOG_ERR,
830                         "eni_recv_stack: NULL buffer, tok = %p\n", tok );
831 #endif
832                 return;
833         }
834
835         /*
836          * Send the data up the stack
837          */
838         STACK_CALL ( CPCS_UNITDATA_SIG, evp->ev_upper,
839                 (void *)evp->ev_toku, evp->ev_connvc, (int)m, 0, err );
840         if ( err ) {
841                 KB_FREEALL ( m );
842         }
843
844         return;
845 }
846