dsynth - Work on count mismatch issues
[dragonfly.git] / usr.bin / dsynth / pkglist.c
1 /*
2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * This code uses concepts and configuration based on 'synth', by
8  * John R. Marino <draco@marino.st>, which was written in ada.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  *
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in
18  *    the documentation and/or other materials provided with the
19  *    distribution.
20  * 3. Neither the name of The DragonFly Project nor the names of its
21  *    contributors may be used to endorse or promote products derived
22  *    from this software without specific, prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  */
37
38 #include "dsynth.h"
39
40 #define PKG_HSIZE       32768
41 #define PKG_HMASK       32767
42
43 static int parsepkglist_file(const char *path, int debugstop);
44 static void childGetPackageInfo(bulk_t *bulk);
45 static void childGetBinaryDistInfo(bulk_t *bulk);
46 static void childOptimizeEnv(bulk_t *bulk);
47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo);
49 static void resolveDepString(pkg_t *pkg, char *depstr,
50                         int gentopo, int dep_type);
51 static pkg_t *processPackageListBulk(int total);
52 static int scan_and_queue_dir(const char *path, const char *level1, int level);
53 static int scan_binary_repo(const char *path);
54 #if 0
55 static void pkgfree(pkg_t *pkg);
56 #endif
57
58 static int PrepareSystemFlag;
59
60 pkg_t *PkgHash1[PKG_HSIZE];     /* by portdir */
61 pkg_t *PkgHash2[PKG_HSIZE];     /* by pkgfile */
62
63 /*
64  * Allocate a new pkg structure plus basic initialization.
65  */
66 static __inline pkg_t *
67 allocpkg(void)
68 {
69         pkg_t *pkg;
70
71         pkg = calloc(1, sizeof(*pkg));
72         pkg->idepon_list.next = &pkg->idepon_list;
73         pkg->idepon_list.prev = &pkg->idepon_list;
74         pkg->deponi_list.next = &pkg->deponi_list;
75         pkg->deponi_list.prev = &pkg->deponi_list;
76
77         return pkg;
78 }
79
80 /*
81  * Simple hash for lookups
82  */
83 static __inline int
84 pkghash(const char *str)
85 {
86         int hv = 0xABC32923;
87         while (*str) {
88                 hv = (hv << 5) ^ *str;
89                 ++str;
90         }
91         hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
92         return (hv & PKG_HMASK);
93 }
94
95 static void
96 pkg_enter(pkg_t *pkg)
97 {
98         pkg_t **pkgp;
99         pkg_t *scan;
100
101         if (pkg->portdir) {
102                 pkgp = &PkgHash1[pkghash(pkg->portdir)];
103                 while ((scan = *pkgp) != NULL) {
104                         if (strcmp(pkg->portdir, scan->portdir) == 0)
105                                 break;
106                         pkgp = &scan->hnext1;
107                 }
108                 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD));
109                 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
110                         ddassert(scan->idepon_list.next == &scan->idepon_list);
111                         ddassert(scan->deponi_list.next == &scan->deponi_list);
112                         *pkgp = pkg;
113                         pkg->hnext1 = scan->hnext1;
114                         free(scan->portdir);
115                         free(scan);
116                         scan = NULL;
117                 }
118                 if (scan == NULL)
119                         *pkgp = pkg;
120         }
121
122         if (pkg->pkgfile) {
123                 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
124                 while ((scan = *pkgp) != NULL) {
125                         if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
126                                 break;
127                         pkgp = &scan->hnext2;
128                 }
129                 if (scan == NULL)
130                         *pkgp = pkg;
131         }
132 }
133
134 static pkg_t *
135 pkg_find(const char *match)
136 {
137         pkg_t **pkgp;
138         pkg_t *pkg;
139
140         pkgp = &PkgHash1[pkghash(match)];
141         for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
142                 if (strcmp(pkg->portdir, match) == 0)
143                         return pkg;
144         }
145         pkgp = &PkgHash2[pkghash(match)];
146         for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
147                 if (strcmp(pkg->pkgfile, match) == 0)
148                         return pkg;
149         }
150         return NULL;
151 }
152
153 /*
154  * Parse a specific list of ports via origin name (portdir/subdir)
155  */
156 pkg_t *
157 ParsePackageList(int n, char **ary, int debugstop)
158 {
159         pkg_t *list;
160         int total;
161         int fail;
162         int i;
163
164         total = 0;
165         fail = 0;
166         initbulk(childGetPackageInfo, MaxBulk);
167
168         /*
169          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
170          * selection, "d" meaning DEBUGSTOP mode, or NULL.
171          */
172         queuebulk("ports-mgmt", "pkg", NULL, "x");
173
174         for (i = 0; i < n; ++i) {
175                 char *l1;
176                 char *l2;
177                 char *l3;
178                 struct stat st;
179
180                 l1 = strdup(ary[i]);
181                 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
182                         total += parsepkglist_file(l1, debugstop);
183                         continue;
184                 }
185
186                 l2 = strchr(l1, '/');
187                 if (l2 == NULL) {
188                         printf("Bad portdir specification: %s\n", l1);
189                         free(l1);
190                         fail = 1;
191                         continue;
192                 }
193                 *l2++ = 0;
194                 l3 = strchr(l2, '@');
195                 if (l3)
196                         *l3++ = 0;
197                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
198                 ++total;
199                 free(l1);
200         }
201         printf("Processing %d ports\n", total);
202
203         list = processPackageListBulk(total);
204         if (fail) {
205                 dfatal("Bad specifications, exiting");
206                 exit(1);
207         }
208
209         return list;
210 }
211
212 static
213 int
214 parsepkglist_file(const char *path, int debugstop)
215 {
216         FILE *fp;
217         char *base;
218         char *l1;
219         char *l2;
220         char *l3;
221         size_t len;
222         int total;
223
224         if ((fp = fopen(path, "r")) == NULL) {
225                 dpanic_errno("Cannot read %s\n", path);
226                 /* NOT REACHED */
227                 return 0;
228         }
229
230         total = 0;
231
232         while ((base = fgetln(fp, &len)) != NULL) {
233                 if (len == 0 || base[len-1] != '\n')
234                         continue;
235                 base[--len] = 0;
236                 l1 = strtok(base, " \t\r\n");
237                 if (l1 == NULL) {
238                         printf("Badly formatted pkg info line: %s\n", base);
239                         continue;
240                 }
241                 l2 = strchr(l1, '/');
242                 if (l2 == NULL) {
243                         printf("Badly formatted specification: %s\n", l1);
244                         continue;
245                 }
246                 *l2++ = 0;
247                 l3 = strchr(l2, '@');
248                 if (l3)
249                         *l3++ = 0;
250                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
251                 ++total;
252         }
253         fclose(fp);
254
255         return total;
256 }
257
258 /*
259  * Parse packages from the list installed on the system.
260  */
261 pkg_t *
262 GetLocalPackageList(void)
263 {
264         pkg_t *list;
265         FILE *fp;
266         char *base;
267         char *data;
268         char *l1;
269         char *l2;
270         char *l3;
271         int total;
272         int state;
273         size_t len;
274
275         PrepareSystemFlag = 1;
276         initbulk(childGetPackageInfo, MaxBulk);
277         total = 0;
278         state = 0;
279         l1 = NULL;
280         l2 = NULL;
281         l3 = NULL;
282
283         fp = popen("pkg info -a -o -A", "r");
284
285         /*
286          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
287          * selection, "d" meaning DEBUGSTOP mode, or NULL.
288          */
289         queuebulk("ports-mgmt", "pkg", NULL, "x");
290
291         while ((base = fgetln(fp, &len)) != NULL) {
292                 if (len == 0 || base[len-1] != '\n')
293                         continue;
294                 base[--len] = 0;
295
296                 data = strchr(base, ':');
297                 if (data == NULL)
298                         continue;
299                 *data++ = 0;
300
301                 base = strtok(base, " \t\r");
302                 data = strtok(data, " \t\r");
303
304                 if (base == NULL || data == NULL)
305                         continue;
306
307                 if (strcmp(base, "Origin") == 0) {
308                         if (state == 1) {
309                                 queuebulk(l1, l2, NULL, NULL);
310                                 state = 0;
311                                 ++total;
312                         }
313
314                         if (strchr(data, '/') == NULL) {
315                                 printf("Badly formatted origin: %s\n", l1);
316                         }
317                         if (l1)
318                                 free(l1);
319                         if (l3)
320                                 free(l3);
321                         l1 = strdup(data);
322                         l2 = strchr(l1, '/');
323                         *l2++ = 0;
324                         l3 = strchr(l2, '@');   /* typically NULL */
325                         if (l3) {
326                                 *l3++ = 0;
327                                 l3 = strdup(l3);
328                         }
329
330                         /*
331                          * Don't queue ports-mgmt/pkg twice, we already
332                          * queued it manually.
333                          */
334                         if (strcmp(l1, "ports-mgmt") != 0 ||
335                             strcmp(l2, "pkg") != 0) {
336                                 state = 1;
337                         }
338                         continue;
339                 }
340                 if (state == 1 && strcmp(base, "flavor") == 0) {
341                         queuebulk(l1, l2, data, NULL);
342                         state = 0;
343                         ++total;
344                 }
345         }
346         if (state == 1) {
347                 queuebulk(l1, l2, NULL, NULL);
348                 /*state = 0; not needed */
349         }
350         if (l1)
351                 free(l1);
352         if (l3)
353                 free(l3);
354
355         pclose(fp);
356
357         printf("Processing %d ports\n", total);
358
359         list = processPackageListBulk(total);
360
361         return list;
362 }
363
364 pkg_t *
365 GetFullPackageList(void)
366 {
367         int total;
368
369         initbulk(childGetPackageInfo, MaxBulk);
370         total = scan_and_queue_dir(DPortsPath, NULL, 1);
371         printf("Scanning %d ports\n", total);
372
373         return processPackageListBulk(total);
374 }
375
376 /*
377  * Caller has queued the process list for bulk operation.  We retrieve
378  * the results and clean up the bulk operation (we may have to do a second
379  * bulk operation so we have to be the ones to clean it up).
380  */
381 static pkg_t *
382 processPackageListBulk(int total)
383 {
384         bulk_t *bulk;
385         pkg_t *scan;
386         pkg_t *list;
387         pkg_t *dep_list;
388         pkg_t **list_tail;
389         int count;
390         int stop_fail;
391         int stop_base_list;
392         int remove_corrupt;
393
394         list = NULL;
395         list_tail = &list;
396         count = 0;
397         remove_corrupt = 0;
398
399         while ((bulk = getbulk()) != NULL) {
400                 ++count;
401                 if ((count & 255) == 0) {
402                         printf("%6.2f%%\r",
403                                 (double)count * 100.0 / (double)total + 0.001);
404                         fflush(stdout);
405                 }
406                 if (bulk->list) {
407                         *list_tail = bulk->list;
408                         bulk->list = NULL;
409                         while ((scan = *list_tail) != NULL) {
410                                 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
411                                         scan->flags |= PKGF_MANUALSEL;
412                                 pkg_enter(scan);
413                                 list_tail = &scan->bnext;
414                         }
415                 }
416                 freebulk(bulk);
417         }
418         printf("100.00%%\n");
419         printf("\nTotal %d\n", count);
420         fflush(stdout);
421
422         /*
423          * Resolve all dependencies for the related packages, potentially
424          * adding anything that could not be found to the list.  This will
425          * continue to issue bulk operations and process the result until
426          * no dependencies are left.
427          */
428         printf("Resolving dependencies...");
429         fflush(stdout);
430         dep_list = list;
431         while (dep_list) {
432                 dep_list = resolveDeps(dep_list, &list_tail, 0);
433         }
434         printf("done\n");
435
436         donebulk();
437
438         /*
439          * Generate the topology
440          */
441         resolveDeps(list, NULL, 1);
442
443         /*
444          * Do a final count, ignore place holders.
445          *
446          * Also set stop_fail if appropriate.  Check for direct specifications
447          * which fail to probe and any direct dependencies of those
448          * specifications, but don't recurse (for now)... don't check indirect
449          * dependencies (i.e. A -> B -> C where A is directly specified, B
450          * is adirect dependency, and C fails to probe).
451          */
452         count = 0;
453         stop_fail = 0;
454         stop_base_list = 0;
455         for (scan = list; scan; scan = scan->bnext) {
456                 if ((scan->flags & PKGF_ERROR) == 0) {
457                         ++count;
458                 }
459                 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) {
460                         pkglink_t *link;
461
462                         /*
463                          * Directly specified package failed to probe
464                          */
465                         if (scan->flags & PKGF_CORRUPT) {
466                                 ++stop_fail;
467                                 ++stop_base_list;
468                         }
469
470                         /*
471                          * Directly specified package had a direct dependency
472                          * that failed to probe (don't go further).
473                          */
474                         PKGLIST_FOREACH(link, &scan->idepon_list) {
475                                 if (link->pkg &&
476                                     (link->pkg->flags & PKGF_CORRUPT)) {
477                                         ++stop_fail;
478                                 }
479                         }
480                 }
481         }
482         printf("Total Returned %d\n", count);
483
484         /*
485          * Check to see if any PKGF_MANUALSEL packages
486          */
487         if (stop_fail) {
488                 printf("%d packages failed to probe\n", stop_fail);
489                 if (PrepareSystemFlag) {
490                         if (stop_fail == stop_base_list) {
491                                 printf(
492   "prepare-system: Some of your installed packages no longer exist in\n"
493   "dports, do you wish to continue rebuilding what does exist?\n");
494                                 if (askyn("Continue anyway? "))
495                                         remove_corrupt = 1;
496                         } else {
497                                 printf(
498   "prepare-system: Some of your installed packages have dependencies\n"
499   "which could not be found in dports, cannot continue, aborting\n");
500                         }
501                 } else {
502                         printf("unable to continue, aborting\n");
503                 }
504                 if (remove_corrupt == 0)
505                         exit(1);
506         }
507
508         /*
509          * Remove corrupt packages before continuing
510          */
511         if (remove_corrupt) {
512                 list_tail = &list;
513                 while ((scan = *list_tail) != NULL) {
514                         if (scan->flags & PKGF_CORRUPT)
515                                 *list_tail = scan->bnext;
516                         else
517                                 list_tail = &scan->bnext;
518                 }
519         }
520
521         /*
522          * Scan our binary distributions and related dependencies looking
523          * for any packages that have already been built.
524          */
525         initbulk(childGetBinaryDistInfo, MaxBulk);
526         total = scan_binary_repo(RepositoryPath);
527         count = 0;
528         printf("Scanning %d packages\n", total);
529
530         while ((bulk = getbulk()) != NULL) {
531                 ++count;
532                 if ((count & 255) == 0) {
533                         printf("%6.2f%%\r",
534                                 (double)count * 100.0 / (double)total + 0.001);
535                         fflush(stdout);
536                 }
537                 freebulk(bulk);
538         }
539         printf("100.00%%\n");
540         printf("\nTotal %d\n", count);
541         fflush(stdout);
542         donebulk();
543
544         printf("all done\n");
545
546         return list;
547 }
548
549 pkg_t *
550 GetPkgPkg(pkg_t **listp)
551 {
552         bulk_t *bulk;
553         pkg_t *scan;
554         pkg_t *s2;
555
556         for (scan = *listp; scan; scan = scan->bnext) {
557                 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
558                         return scan;
559         }
560
561         /*
562          * This will force pkg to be built, but generally this code
563          * is not reached because the package list processing code
564          * adds ports-mgmt/pkg unconditionally.
565          */
566         initbulk(childGetPackageInfo, MaxBulk);
567         queuebulk("ports-mgmt", "pkg", NULL, "x");
568         bulk = getbulk();
569         dassert(bulk, "Cannot find ports-mgmt/pkg");
570         scan = bulk->list;
571         bulk->list = NULL;
572         freebulk(bulk);
573         donebulk();
574
575
576         /*
577          * Include added packages to the total and add the initial bulk
578          * built packages to the list so they get counted.
579          */
580         for (s2 = scan; s2->bnext; s2 = s2->bnext)
581                 ++BuildTotal;
582         for (s2 = scan; s2->bnext; s2 = s2->bnext)
583                 ;
584         s2->bnext = *listp;
585         *listp = scan;
586         ++BuildTotal;
587
588         return scan;
589 }
590
591 /*
592  * Try to optimize the environment by supplying information that
593  * the ports system would generally have to run stuff to get on
594  * every package.
595  *
596  * See childOptimizeEnv() for the actual handling.  We execute
597  * a single make -V... -V... for ports-mgmt/pkg from within the
598  * bulk system (which handles the environment and disables
599  * /etc/make.conf), and we then call addbuildenv() as appropriate.
600  *
601  * _PERL5_FROM_BIN
602  * add others...
603  */
604 void
605 OptimizeEnv(void)
606 {
607         bulk_t *bulk;
608
609         initbulk(childOptimizeEnv, MaxBulk);
610         queuebulk("ports-mgmt", "pkg", NULL, NULL);
611         bulk = getbulk();
612         freebulk(bulk);
613         donebulk();
614 }
615
616 /*
617  * Run through the list resolving dependencies and constructing the topology
618  * linkages.   This may append packages to the list.  Dependencies to dummy
619  * nodes which do not specify a flavor do not need special handling, the
620  * search code in build.c will properly follow the first flavor.
621  */
622 static pkg_t *
623 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
624 {
625         pkg_t *ret_list = NULL;
626         pkg_t *scan;
627         pkg_t *use;
628         bulk_t *bulk;
629
630         for (scan = list; scan; scan = scan->bnext) {
631                 use = pkg_find(scan->portdir);
632                 resolveFlavors(use, scan->flavors, gentopo);
633                 resolveDepString(use, scan->fetch_deps,
634                                  gentopo, DEP_TYPE_FETCH);
635                 resolveDepString(use, scan->ext_deps,
636                                  gentopo, DEP_TYPE_EXT);
637                 resolveDepString(use, scan->patch_deps,
638                                  gentopo, DEP_TYPE_PATCH);
639                 resolveDepString(use, scan->build_deps,
640                                  gentopo, DEP_TYPE_BUILD);
641                 resolveDepString(use, scan->lib_deps,
642                                  gentopo, DEP_TYPE_LIB);
643                 resolveDepString(use, scan->run_deps,
644                                  gentopo, DEP_TYPE_RUN);
645         }
646
647         /*
648          * No bulk ops are queued when doing the final topology
649          * generation.
650          *
651          * Avoid entering duplicate results from the bulk ops.  Duplicate
652          * results are mostly filtered out, but not always.  A dummy node
653          * representing multiple flavors will parse-out the flavors
654          */
655         if (gentopo)
656                 return NULL;
657         while ((bulk = getbulk()) != NULL) {
658                 if (bulk->list) {
659                         if (ret_list == NULL)
660                                 ret_list = bulk->list;
661                         **list_tailp = bulk->list;
662                         bulk->list = NULL;
663                         while ((scan = **list_tailp) != NULL) {
664                                 pkg_enter(scan);
665                                 *list_tailp = &scan->bnext;
666                         }
667                 }
668                 freebulk(bulk);
669         }
670         return (ret_list);
671 }
672
673 /*
674  * Resolve a generic node that has flavors, queue to retrieve info for
675  * each flavor and setup linkages as appropriate.
676  */
677 static void
678 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo)
679 {
680         char *flavor_base;
681         char *flavor_scan;
682         char *flavor;
683         char *portdir;
684         char *s1;
685         char *s2;
686         pkg_t *dpkg;
687         pkglink_t *link;
688
689         if ((pkg->flags & PKGF_DUMMY) == 0)
690                 return;
691         if (pkg->flavors == NULL || pkg->flavors[0] == 0)
692                 return;
693         flavor_base = strdup(flavors);
694         flavor_scan = flavor_base;
695
696         for (;;) {
697                 do {
698                         flavor = strsep(&flavor_scan, " \t");
699                 } while (flavor && *flavor == 0);
700                 if (flavor == NULL)
701                         break;
702
703                 /*
704                  * Iterate each flavor generating "s1/s2@flavor".
705                  *
706                  * queuebulk() info for each flavor, and set-up the
707                  * linkages in the topology generation pass.
708                  */
709                 asprintf(&portdir, "%s@%s", pkg->portdir, flavor);
710                 s1 = strdup(pkg->portdir);
711                 s2 = strchr(s1, '/');
712                 *s2++ = 0;
713
714                 dpkg = pkg_find(portdir);
715                 if (dpkg && gentopo) {
716                         /*
717                          * Setup linkages
718                          */
719                         free(portdir);
720
721                         link = calloc(1, sizeof(*link));
722                         link->pkg = dpkg;
723                         link->next = &pkg->idepon_list;
724                         link->prev = pkg->idepon_list.prev;
725                         link->next->prev = link;
726                         link->prev->next = link;
727                         link->dep_type = DEP_TYPE_BUILD;
728
729                         link = calloc(1, sizeof(*link));
730                         link->pkg = pkg;
731                         link->next = &dpkg->deponi_list;
732                         link->prev = dpkg->deponi_list.prev;
733                         link->next->prev = link;
734                         link->prev->next = link;
735                         link->dep_type = DEP_TYPE_BUILD;
736                         ++dpkg->depi_count;
737                 } else if (gentopo == 0 && dpkg == NULL) {
738                         /*
739                          * Use a place-holder to prevent duplicate
740                          * dependencies from being processed.  The placeholder
741                          * will be replaced by the actual dependency.
742                          */
743                         dpkg = allocpkg();
744                         dpkg->portdir = portdir;
745                         dpkg->flags = PKGF_PLACEHOLD;
746                         pkg_enter(dpkg);
747                         queuebulk(s1, s2, flavor, NULL);
748                 }
749                 free(s1);
750         }
751         free(flavor_base);
752 }
753
754 static void
755 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
756 {
757         char *copy_base;
758         char *copy;
759         char *dep;
760         char *sep;
761         char *tag;
762         char *flavor;
763         pkg_t *dpkg;
764
765         if (depstr == NULL || depstr[0] == 0)
766                 return;
767
768         copy_base = strdup(depstr);
769         copy = copy_base;
770
771         for (;;) {
772                 do {
773                         dep = strsep(&copy, " \t");
774                 } while (dep && *dep == 0);
775                 if (dep == NULL)
776                         break;
777
778                 /*
779                  * Ignore dependencies prefixed with ${NONEXISTENT}
780                  */
781                 if (strncmp(dep, "/nonexistent:", 13) == 0)
782                         continue;
783
784                 dep = strchr(dep, ':');
785                 if (dep == NULL || *dep != ':') {
786                         printf("Error parsing dependency for %s: %s\n",
787                                pkg->portdir, copy_base);
788                         continue;
789                 }
790                 ++dep;
791
792                 /*
793                  * Strip-off any DPortsPath prefix.  EXTRACT_DEPENDS
794                  * often (always?) generates this prefix.
795                  */
796                 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
797                         dep += strlen(DPortsPath);
798                         if (*dep == '/')
799                                 ++dep;
800                 }
801
802                 /*
803                  * Strip-off any tag (such as :patch).  We don't try to
804                  * organize dependencies at this fine a grain (for now).
805                  */
806                 tag = strchr(dep, ':');
807                 if (tag)
808                         *tag++ = 0;
809
810                 /*
811                  * Locate the dependency
812                  */
813                 if ((dpkg = pkg_find(dep)) != NULL) {
814                         if (gentopo) {
815                                 pkglink_t *link;
816
817                                 /*
818                                  * NOTE: idep_count is calculated recursively
819                                  *       at build-time
820                                  */
821                                 ddprintf(0, "Add Dependency %s -> %s\n",
822                                         pkg->portdir, dpkg->portdir);
823                                 link = calloc(1, sizeof(*link));
824                                 link->pkg = dpkg;
825                                 link->next = &pkg->idepon_list;
826                                 link->prev = pkg->idepon_list.prev;
827                                 link->next->prev = link;
828                                 link->prev->next = link;
829                                 link->dep_type = dep_type;
830
831                                 link = calloc(1, sizeof(*link));
832                                 link->pkg = pkg;
833                                 link->next = &dpkg->deponi_list;
834                                 link->prev = dpkg->deponi_list.prev;
835                                 link->next->prev = link;
836                                 link->prev->next = link;
837                                 link->dep_type = dep_type;
838                                 ++dpkg->depi_count;
839                         }
840                         continue;
841                 }
842
843                 /*
844                  * This shouldn't happen because we already took a first
845                  * pass and should have generated the pkgs.
846                  */
847                 if (gentopo) {
848                         printf("Topology Generate failed for %s: %s\n",
849                                 pkg->portdir, copy_base);
850                         continue;
851                 }
852
853                 /*
854                  * Separate out the two dports directory components and
855                  * extract the optional '@flavor' specification.
856                  */
857                 sep = strchr(dep, '/');
858                 if (sep == NULL) {
859                         printf("Error parsing dependency for %s: %s\n",
860                                pkg->portdir, copy_base);
861                         continue;
862                 }
863                 *sep++ = 0;
864
865                 /*
866                  * The flavor hangs off the separator, not the tag
867                  */
868                 flavor = strrchr(sep, '@');
869 #if 0
870                 if (tag)
871                         flavor = strrchr(tag, '@');
872                 else
873                         flavor = strrchr(sep, '@');
874 #endif
875                 if (flavor)
876                         *flavor++ = 0;
877
878                 if (flavor)
879                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
880                                pkg->portdir, dep, sep, flavor);
881                 else
882                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
883                                pkg->portdir, dep, sep);
884
885                 /*
886                  * Use a place-holder to prevent duplicate dependencies from
887                  * being processed.  The placeholder will be replaced by
888                  * the actual dependency.
889                  */
890                 dpkg = allocpkg();
891                 if (flavor)
892                         asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
893                 else
894                         asprintf(&dpkg->portdir, "%s/%s", dep, sep);
895                 dpkg->flags = PKGF_PLACEHOLD;
896                 pkg_enter(dpkg);
897
898                 queuebulk(dep, sep, flavor, NULL);
899         }
900         free(copy_base);
901 }
902
903 void
904 FreePackageList(pkg_t *pkgs __unused)
905 {
906         dfatal("not implemented");
907 }
908
909 /*
910  * Scan some or all dports to allocate the related pkg structure.  Dependencies
911  * are stored but not processed.
912  *
913  * Threaded function
914  */
915 static void
916 childGetPackageInfo(bulk_t *bulk)
917 {
918         pkg_t *pkg;
919         char *flavor;
920         char *ptr;
921         FILE *fp;
922         int line;
923         size_t len;
924         char *portpath;
925         char *flavarg;
926         const char *cav[MAXCAC];
927         pid_t pid;
928         int cac;
929
930         /*
931          * If the package has flavors we will loop on each one.  If a flavor
932          * is not passed in s3 we will loop on all flavors, otherwise we will
933          * only process the passed-in flavor.
934          */
935         flavor = bulk->s3;      /* usually NULL */
936
937         bulk->list = NULL;
938
939         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
940         if (flavor)
941                 asprintf(&flavarg, "FLAVOR=%s", flavor);
942         else
943                 flavarg = NULL;
944
945         cac = 0;
946         cav[cac++] = MAKE_BINARY;
947         cav[cac++] = "-C";
948         cav[cac++] = portpath;
949         if (flavarg)
950                 cav[cac++] = flavarg;
951         cav[cac++] = "-VPKGVERSION";
952         cav[cac++] = "-VPKGFILE:T";
953         cav[cac++] = "-VALLFILES";
954         cav[cac++] = "-VDIST_SUBDIR";
955         cav[cac++] = "-VMAKE_JOBS_NUMBER";
956         cav[cac++] = "-VIGNORE";
957         cav[cac++] = "-VFETCH_DEPENDS";
958         cav[cac++] = "-VEXTRACT_DEPENDS";
959         cav[cac++] = "-VPATCH_DEPENDS";
960         cav[cac++] = "-VBUILD_DEPENDS";
961         cav[cac++] = "-VLIB_DEPENDS";
962         cav[cac++] = "-VRUN_DEPENDS";
963         cav[cac++] = "-VSELECTED_OPTIONS";
964         cav[cac++] = "-VDESELECTED_OPTIONS";
965         cav[cac++] = "-VUSE_LINUX";
966         cav[cac++] = "-VFLAVORS";
967         cav[cac++] = "-VUSES";
968
969         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
970         free(portpath);
971         freestrp(&flavarg);
972
973         pkg = allocpkg();
974         if (flavor)
975                 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
976         else
977                 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
978
979         line = 1;
980         while ((ptr = fgetln(fp, &len)) != NULL) {
981                 if (len == 0 || ptr[len-1] != '\n') {
982                         dfatal("Bad package info for %s/%s response line %d",
983                                bulk->s1, bulk->s2, line);
984                 }
985                 ptr[--len] = 0;
986
987                 switch(line) {
988                 case 1:         /* PKGVERSION */
989                         asprintf(&pkg->version, "%s", ptr);
990                         break;
991                 case 2:         /* PKGFILE */
992                         asprintf(&pkg->pkgfile, "%s", ptr);
993                         break;
994                 case 3:         /* ALLFILES (aka DISTFILES + patch files) */
995                         asprintf(&pkg->distfiles, "%s", ptr);
996                         break;
997                 case 4:         /* DIST_SUBDIR */
998                         pkg->distsubdir = strdup_or_null(ptr);
999                         break;
1000                 case 5:         /* MAKE_JOBS_NUMBER */
1001                         pkg->make_jobs_number = strtol(ptr, NULL, 0);
1002                         break;
1003                 case 6:         /* IGNORE */
1004                         pkg->ignore = strdup_or_null(ptr);
1005                         break;
1006                 case 7:         /* FETCH_DEPENDS */
1007                         pkg->fetch_deps = strdup_or_null(ptr);
1008                         break;
1009                 case 8:         /* EXTRACT_DEPENDS */
1010                         pkg->ext_deps = strdup_or_null(ptr);
1011                         break;
1012                 case 9:         /* PATCH_DEPENDS */
1013                         pkg->patch_deps = strdup_or_null(ptr);
1014                         break;
1015                 case 10:        /* BUILD_DEPENDS */
1016                         pkg->build_deps = strdup_or_null(ptr);
1017                         break;
1018                 case 11:        /* LIB_DEPENDS */
1019                         pkg->lib_deps = strdup_or_null(ptr);
1020                         break;
1021                 case 12:        /* RUN_DEPENDS */
1022                         pkg->run_deps = strdup_or_null(ptr);
1023                         break;
1024                 case 13:        /* SELECTED_OPTIONS */
1025                         pkg->pos_options = strdup_or_null(ptr);
1026                         break;
1027                 case 14:        /* DESELECTED_OPTIONS */
1028                         pkg->neg_options = strdup_or_null(ptr);
1029                         break;
1030                 case 15:        /* USE_LINUX */
1031                         if (ptr[0])
1032                                 pkg->use_linux = 1;
1033                         break;
1034                 case 16:        /* FLAVORS */
1035                         asprintf(&pkg->flavors, "%s", ptr);
1036                         break;
1037                 case 17:        /* USES */
1038                         asprintf(&pkg->uses, "%s", ptr);
1039                         if (strstr(pkg->uses, "metaport"))
1040                                 pkg->flags |= PKGF_META;
1041                         break;
1042                 default:
1043                         printf("EXTRA LINE: %s\n", ptr);
1044                         break;
1045                 }
1046                 ++line;
1047         }
1048         if (line == 1) {
1049                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1050                 pkg->flags |= PKGF_NOTFOUND;
1051         } else if (line != 17 + 1) {
1052                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1053                 pkg->flags |= PKGF_CORRUPT;
1054         }
1055         if (dexec_close(fp, pid)) {
1056                 printf("make -V* command for %s/%s failed\n",
1057                         bulk->s1, bulk->s2);
1058                 pkg->flags |= PKGF_CORRUPT;
1059         }
1060         ddassert(bulk->s1);
1061
1062         /*
1063          * DEBUGSTOP mode
1064          */
1065         if (bulk->s4 && bulk->s4[0] == 'd')
1066                 pkg->flags |= PKGF_DEBUGSTOP;
1067
1068         /*
1069          * Mark as a dummy node, the front-end will iterate the flavors
1070          * and create sub-nodes for us.
1071          *
1072          * Get rid of elements returned that are for the first flavor.
1073          * We are creating a dummy node here, not the node for the first
1074          * flavor.
1075          */
1076         if (flavor == NULL && pkg->flavors && pkg->flavors[0]) {
1077                 pkg->flags |= PKGF_DUMMY;
1078                 freestrp(&pkg->fetch_deps);
1079                 freestrp(&pkg->ext_deps);
1080                 freestrp(&pkg->patch_deps);
1081                 freestrp(&pkg->build_deps);
1082                 freestrp(&pkg->lib_deps);
1083                 freestrp(&pkg->run_deps);
1084                 freestrp(&pkg->pkgfile);
1085         }
1086
1087         /*
1088          * Only one pkg is put on the return list now.  This code no
1089          * longer creates pseudo-nodes for flavors (the frontend requests
1090          * each flavor instead).
1091          */
1092         bulk->list = pkg;
1093 }
1094
1095 /*
1096  * Query the package (at least to make sure it hasn't been truncated)
1097  * and mark it as PACKAGED if found.
1098  *
1099  * Threaded function
1100  */
1101 static void
1102 childGetBinaryDistInfo(bulk_t *bulk)
1103 {
1104         char *ptr;
1105         FILE *fp;
1106         size_t len;
1107         pkg_t *pkg;
1108         const char *cav[MAXCAC];
1109         char *repopath;
1110         char buf[1024];
1111         pid_t pid;
1112         int cac;
1113         int deleteme;
1114
1115         asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
1116
1117         cac = 0;
1118         cav[cac++] = PKG_BINARY;
1119         cav[cac++] = "query";
1120         cav[cac++] = "-F";
1121         cav[cac++] = repopath;
1122         cav[cac++] = "%n-%v";
1123
1124         fp = dexec_open(cav, cac, &pid, NULL, 1, 0);
1125         deleteme = DeleteObsoletePkgs;
1126
1127         while ((ptr = fgetln(fp, &len)) != NULL) {
1128                 if (len == 0 || ptr[len-1] != '\n')
1129                         continue;
1130                 ptr[len-1] = 0;
1131                 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
1132
1133                 pkg = pkg_find(buf);
1134                 if (pkg) {
1135                         pkg->flags |= PKGF_PACKAGED;
1136                         deleteme = 0;
1137                 } else {
1138                         ddprintf(0, "Note: package scan, not in list, "
1139                                     "skipping %s\n", buf);
1140                 }
1141         }
1142         if (dexec_close(fp, pid)) {
1143                 printf("pkg query command failed for %s\n", repopath);
1144         }
1145         if (deleteme) {
1146                 dlog(DLOG_ALL | DLOG_STDOUT,
1147                      "Deleting obsolete package %s\n", repopath);
1148                 remove(repopath);
1149         }
1150         free(repopath);
1151 }
1152
1153 static void
1154 childOptimizeEnv(bulk_t *bulk)
1155 {
1156         char *portpath;
1157         char *ptr;
1158         FILE *fp;
1159         int line;
1160         size_t len;
1161         const char *cav[MAXCAC];
1162         pid_t pid;
1163         int cac;
1164
1165         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
1166
1167         cac = 0;
1168         cav[cac++] = MAKE_BINARY;
1169         cav[cac++] = "-C";
1170         cav[cac++] = portpath;
1171         cav[cac++] = "-V_PERL5_FROM_BIN";
1172
1173         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
1174         free(portpath);
1175
1176         line = 1;
1177         while ((ptr = fgetln(fp, &len)) != NULL) {
1178                 if (len == 0 || ptr[len-1] != '\n') {
1179                         dfatal("Bad package info for %s/%s response line %d",
1180                                bulk->s1, bulk->s2, line);
1181                 }
1182                 ptr[--len] = 0;
1183
1184                 switch(line) {
1185                 case 1:         /* _PERL5_FROM_BIN */
1186                         addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1187                         break;
1188                 default:
1189                         printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1190                         break;
1191                 }
1192                 ++line;
1193         }
1194         if (line == 1) {
1195                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1196         } else if (line != 1 + 1) {
1197                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1198         }
1199         if (dexec_close(fp, pid)) {
1200                 printf("childOptimizeEnv() failed\n");
1201         }
1202 }
1203
1204 static int
1205 scan_and_queue_dir(const char *path, const char *level1, int level)
1206 {
1207         DIR *dir;
1208         char *s1;
1209         char *s2;
1210         struct dirent *den;
1211         struct stat st;
1212         int count = 0;
1213
1214         dir = opendir(path);
1215         dassert(dir, "Cannot open dports path \"%s\"", path);
1216
1217         while ((den = readdir(dir)) != NULL) {
1218                 if (den->d_namlen == 1 && den->d_name[0] == '.')
1219                         continue;
1220                 if (den->d_namlen == 2 &&
1221                     den->d_name[0] == '.' && den->d_name[1] == '.')
1222                         continue;
1223                 asprintf(&s1, "%s/%s", path, den->d_name);
1224                 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1225                         free(s1);
1226                         continue;
1227                 }
1228                 if (level == 1) {
1229                         count += scan_and_queue_dir(s1, den->d_name, 2);
1230                         free(s1);
1231                         continue;
1232                 }
1233                 asprintf(&s2, "%s/Makefile", s1);
1234                 if (lstat(s2, &st) == 0) {
1235                         queuebulk(level1, den->d_name, NULL, NULL);
1236                         ++count;
1237                 }
1238                 free(s1);
1239                 free(s2);
1240         }
1241         closedir(dir);
1242
1243         return count;
1244 }
1245
1246 static int
1247 scan_binary_repo(const char *path)
1248 {
1249         DIR *dir;
1250         struct dirent *den;
1251         size_t len;
1252         int count;
1253
1254         count = 0;
1255         dir = opendir(path);
1256         dassert(dir, "Cannot open repository path \"%s\"", path);
1257
1258         /*
1259          * NOTE: Test includes the '.' in the suffix.
1260          */
1261         while ((den = readdir(dir)) != NULL) {
1262                 len = strlen(den->d_name);
1263                 if (len > 4 &&
1264                     strcmp(den->d_name + len - 4, UsePkgSufx) == 0) {
1265                         queuebulk(den->d_name, NULL, NULL, NULL);
1266                         ++count;
1267                 }
1268         }
1269         closedir(dir);
1270
1271         return count;
1272 }
1273
1274 #if 0
1275 static void
1276 pkgfree(pkg_t *pkg)
1277 {
1278         freestrp(&pkg->portdir);
1279         freestrp(&pkg->version);
1280         freestrp(&pkg->pkgfile);
1281         freestrp(&pkg->ignore);
1282         freestrp(&pkg->fetch_deps);
1283         freestrp(&pkg->ext_deps);
1284         freestrp(&pkg->patch_deps);
1285         freestrp(&pkg->build_deps);
1286         freestrp(&pkg->lib_deps);
1287         freestrp(&pkg->run_deps);
1288         freestrp(&pkg->pos_options);
1289         freestrp(&pkg->neg_options);
1290         freestrp(&pkg->flavors);
1291         free(pkg);
1292 }
1293 #endif