usr.bin/dsynth/pkglist.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37
  38 #include "dsynth.h"
  39
  40 #define PKG_HSIZE       32768
  41 #define PKG_HMASK       32767
  42
  43 static int parsepkglist_file(const char *path, int debugstop);
  44 static void childGetPackageInfo(bulk_t *bulk);
  45 static void childGetBinaryDistInfo(bulk_t *bulk);
  46 static void childOptimizeEnv(bulk_t *bulk);
  47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
  48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo);
  49 static void resolveDepString(pkg_t *pkg, char *depstr,
  50                         int gentopo, int dep_type);
  51 static pkg_t *processPackageListBulk(int total);
  52 static int scan_and_queue_dir(const char *path, const char *level1, int level);
  53 static int scan_binary_repo(const char *path);
  54 #if 0
  55 static void pkgfree(pkg_t *pkg);
  56 #endif
  57
  58 static int PrepareSystemFlag;
  59
  60 pkg_t *PkgHash1[PKG_HSIZE];     /* by portdir */
  61 pkg_t *PkgHash2[PKG_HSIZE];     /* by pkgfile */
  62
  63 /*
  64  * Allocate a new pkg structure plus basic initialization.
  65  */
  66 static __inline pkg_t *
  67 allocpkg(void)
  68 {
  69         pkg_t *pkg;
  70
  71         pkg = calloc(1, sizeof(*pkg));
  72         pkg->idepon_list.next = &pkg->idepon_list;
  73         pkg->idepon_list.prev = &pkg->idepon_list;
  74         pkg->deponi_list.next = &pkg->deponi_list;
  75         pkg->deponi_list.prev = &pkg->deponi_list;
  76
  77         return pkg;
  78 }
  79
  80 /*
  81  * Simple hash for lookups
  82  */
  83 static __inline int
  84 pkghash(const char *str)
  85 {
  86         int hv = 0xABC32923;
  87         while (*str) {
  88                 hv = (hv << 5) ^ *str;
  89                 ++str;
  90         }
  91         hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
  92         return (hv & PKG_HMASK);
  93 }
  94
  95 static void
  96 pkg_enter(pkg_t *pkg)
  97 {
  98         pkg_t **pkgp;
  99         pkg_t *scan;
 100
 101         if (pkg->portdir) {
 102                 pkgp = &PkgHash1[pkghash(pkg->portdir)];
 103                 while ((scan = *pkgp) != NULL) {
 104                         if (strcmp(pkg->portdir, scan->portdir) == 0)
 105                                 break;
 106                         pkgp = &scan->hnext1;
 107                 }
 108                 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD));
 109                 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
 110                         ddassert(scan->idepon_list.next == &scan->idepon_list);
 111                         ddassert(scan->deponi_list.next == &scan->deponi_list);
 112                         *pkgp = pkg;
 113                         pkg->hnext1 = scan->hnext1;
 114                         free(scan->portdir);
 115                         free(scan);
 116                         scan = NULL;
 117                 }
 118                 if (scan == NULL)
 119                         *pkgp = pkg;
 120         }
 121
 122         if (pkg->pkgfile) {
 123                 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
 124                 while ((scan = *pkgp) != NULL) {
 125                         if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
 126                                 break;
 127                         pkgp = &scan->hnext2;
 128                 }
 129                 if (scan == NULL)
 130                         *pkgp = pkg;
 131         }
 132 }
 133
 134 static pkg_t *
 135 pkg_find(const char *match)
 136 {
 137         pkg_t **pkgp;
 138         pkg_t *pkg;
 139
 140         pkgp = &PkgHash1[pkghash(match)];
 141         for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
 142                 if (strcmp(pkg->portdir, match) == 0)
 143                         return pkg;
 144         }
 145         pkgp = &PkgHash2[pkghash(match)];
 146         for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
 147                 if (strcmp(pkg->pkgfile, match) == 0)
 148                         return pkg;
 149         }
 150         return NULL;
 151 }
 152
 153 /*
 154  * Parse a specific list of ports via origin name (portdir/subdir)
 155  */
 156 pkg_t *
 157 ParsePackageList(int n, char **ary, int debugstop)
 158 {
 159         pkg_t *list;
 160         int total;
 161         int fail;
 162         int i;
 163
 164         total = 0;
 165         fail = 0;
 166         initbulk(childGetPackageInfo, MaxBulk);
 167
 168         /*
 169          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 170          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 171          */
 172         queuebulk("ports-mgmt", "pkg", NULL, "x");
 173
 174         for (i = 0; i < n; ++i) {
 175                 char *l1;
 176                 char *l2;
 177                 char *l3;
 178                 struct stat st;
 179
 180                 l1 = strdup(ary[i]);
 181                 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
 182                         total += parsepkglist_file(l1, debugstop);
 183                         continue;
 184                 }
 185
 186                 l2 = strchr(l1, '/');
 187                 if (l2 == NULL) {
 188                         printf("Bad portdir specification: %s\n", l1);
 189                         free(l1);
 190                         fail = 1;
 191                         continue;
 192                 }
 193                 *l2++ = 0;
 194                 l3 = strchr(l2, '@');
 195                 if (l3)
 196                         *l3++ = 0;
 197                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 198                 ++total;
 199                 free(l1);
 200         }
 201         printf("Processing %d ports\n", total);
 202
 203         list = processPackageListBulk(total);
 204         if (fail) {
 205                 dfatal("Bad specifications, exiting");
 206                 exit(1);
 207         }
 208
 209         return list;
 210 }
 211
 212 static
 213 int
 214 parsepkglist_file(const char *path, int debugstop)
 215 {
 216         FILE *fp;
 217         char *base;
 218         char *l1;
 219         char *l2;
 220         char *l3;
 221         size_t len;
 222         int total;
 223
 224         if ((fp = fopen(path, "r")) == NULL) {
 225                 dpanic_errno("Cannot read %s\n", path);
 226                 /* NOT REACHED */
 227                 return 0;
 228         }
 229
 230         total = 0;
 231
 232         while ((base = fgetln(fp, &len)) != NULL) {
 233                 if (len == 0 || base[len-1] != '\n')
 234                         continue;
 235                 base[--len] = 0;
 236                 l1 = strtok(base, " \t\r\n");
 237                 if (l1 == NULL) {
 238                         printf("Badly formatted pkg info line: %s\n", base);
 239                         continue;
 240                 }
 241                 l2 = strchr(l1, '/');
 242                 if (l2 == NULL) {
 243                         printf("Badly formatted specification: %s\n", l1);
 244                         continue;
 245                 }
 246                 *l2++ = 0;
 247                 l3 = strchr(l2, '@');
 248                 if (l3)
 249                         *l3++ = 0;
 250                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 251                 ++total;
 252         }
 253         fclose(fp);
 254
 255         return total;
 256 }
 257
 258 /*
 259  * Parse packages from the list installed on the system.
 260  */
 261 pkg_t *
 262 GetLocalPackageList(void)
 263 {
 264         pkg_t *list;
 265         FILE *fp;
 266         char *base;
 267         char *data;
 268         char *l1;
 269         char *l2;
 270         char *l3;
 271         int total;
 272         int state;
 273         size_t len;
 274
 275         PrepareSystemFlag = 1;
 276         initbulk(childGetPackageInfo, MaxBulk);
 277         total = 0;
 278         state = 0;
 279         l1 = NULL;
 280         l2 = NULL;
 281         l3 = NULL;
 282
 283         fp = popen("pkg info -a -o -A", "r");
 284
 285         /*
 286          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 287          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 288          */
 289         queuebulk("ports-mgmt", "pkg", NULL, "x");
 290
 291         while ((base = fgetln(fp, &len)) != NULL) {
 292                 if (len == 0 || base[len-1] != '\n')
 293                         continue;
 294                 base[--len] = 0;
 295
 296                 data = strchr(base, ':');
 297                 if (data == NULL)
 298                         continue;
 299                 *data++ = 0;
 300
 301                 base = strtok(base, " \t\r");
 302                 data = strtok(data, " \t\r");
 303
 304                 if (base == NULL || data == NULL)
 305                         continue;
 306
 307                 if (strcmp(base, "Origin") == 0) {
 308                         if (state == 1) {
 309                                 queuebulk(l1, l2, NULL, NULL);
 310                                 state = 0;
 311                                 ++total;
 312                         }
 313
 314                         if (strchr(data, '/') == NULL) {
 315                                 printf("Badly formatted origin: %s\n", l1);
 316                         }
 317                         if (l1)
 318                                 free(l1);
 319                         if (l3)
 320                                 free(l3);
 321                         l1 = strdup(data);
 322                         l2 = strchr(l1, '/');
 323                         *l2++ = 0;
 324                         l3 = strchr(l2, '@');   /* typically NULL */
 325                         if (l3) {
 326                                 *l3++ = 0;
 327                                 l3 = strdup(l3);
 328                         }
 329
 330                         /*
 331                          * Don't queue ports-mgmt/pkg twice, we already
 332                          * queued it manually.
 333                          */
 334                         if (strcmp(l1, "ports-mgmt") != 0 ||
 335                             strcmp(l2, "pkg") != 0) {
 336                                 state = 1;
 337                         }
 338                         continue;
 339                 }
 340                 if (state == 1 && strcmp(base, "flavor") == 0) {
 341                         queuebulk(l1, l2, data, NULL);
 342                         state = 0;
 343                         ++total;
 344                 }
 345         }
 346         if (state == 1) {
 347                 queuebulk(l1, l2, NULL, NULL);
 348                 /*state = 0; not needed */
 349         }
 350         if (l1)
 351                 free(l1);
 352         if (l3)
 353                 free(l3);
 354
 355         pclose(fp);
 356
 357         printf("Processing %d ports\n", total);
 358
 359         list = processPackageListBulk(total);
 360
 361         return list;
 362 }
 363
 364 pkg_t *
 365 GetFullPackageList(void)
 366 {
 367         int total;
 368
 369         initbulk(childGetPackageInfo, MaxBulk);
 370         total = scan_and_queue_dir(DPortsPath, NULL, 1);
 371         printf("Scanning %d ports\n", total);
 372
 373         return processPackageListBulk(total);
 374 }
 375
 376 /*
 377  * Caller has queued the process list for bulk operation.  We retrieve
 378  * the results and clean up the bulk operation (we may have to do a second
 379  * bulk operation so we have to be the ones to clean it up).
 380  */
 381 static pkg_t *
 382 processPackageListBulk(int total)
 383 {
 384         bulk_t *bulk;
 385         pkg_t *scan;
 386         pkg_t *list;
 387         pkg_t *dep_list;
 388         pkg_t **list_tail;
 389         int count;
 390         int stop_fail;
 391         int stop_base_list;
 392         int remove_corrupt;
 393
 394         list = NULL;
 395         list_tail = &list;
 396         count = 0;
 397         remove_corrupt = 0;
 398
 399         while ((bulk = getbulk()) != NULL) {
 400                 ++count;
 401                 if ((count & 255) == 0) {
 402                         printf("%6.2f%%\r",
 403                                 (double)count * 100.0 / (double)total + 0.001);
 404                         fflush(stdout);
 405                 }
 406                 if (bulk->list) {
 407                         *list_tail = bulk->list;
 408                         bulk->list = NULL;
 409                         while ((scan = *list_tail) != NULL) {
 410                                 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
 411                                         scan->flags |= PKGF_MANUALSEL;
 412                                 pkg_enter(scan);
 413                                 list_tail = &scan->bnext;
 414                         }
 415                 }
 416                 freebulk(bulk);
 417         }
 418         printf("100.00%%\n");
 419         printf("\nTotal %d\n", count);
 420         fflush(stdout);
 421
 422         /*
 423          * Resolve all dependencies for the related packages, potentially
 424          * adding anything that could not be found to the list.  This will
 425          * continue to issue bulk operations and process the result until
 426          * no dependencies are left.
 427          */
 428         printf("Resolving dependencies...");
 429         fflush(stdout);
 430         dep_list = list;
 431         while (dep_list) {
 432                 dep_list = resolveDeps(dep_list, &list_tail, 0);
 433         }
 434         printf("done\n");
 435
 436         donebulk();
 437
 438         /*
 439          * Generate the topology
 440          */
 441         resolveDeps(list, NULL, 1);
 442
 443         /*
 444          * Do a final count, ignore place holders.
 445          *
 446          * Also set stop_fail if appropriate.  Check for direct specifications
 447          * which fail to probe and any direct dependencies of those
 448          * specifications, but don't recurse (for now)... don't check indirect
 449          * dependencies (i.e. A -> B -> C where A is directly specified, B
 450          * is adirect dependency, and C fails to probe).
 451          */
 452         count = 0;
 453         stop_fail = 0;
 454         stop_base_list = 0;
 455         for (scan = list; scan; scan = scan->bnext) {
 456                 if ((scan->flags & PKGF_ERROR) == 0) {
 457                         ++count;
 458                 }
 459                 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) {
 460                         pkglink_t *link;
 461
 462                         /*
 463                          * Directly specified package failed to probe
 464                          */
 465                         if (scan->flags & PKGF_CORRUPT) {
 466                                 ++stop_fail;
 467                                 ++stop_base_list;
 468                         }
 469
 470                         /*
 471                          * Directly specified package had a direct dependency
 472                          * that failed to probe (don't go further).
 473                          */
 474                         PKGLIST_FOREACH(link, &scan->idepon_list) {
 475                                 if (link->pkg &&
 476                                     (link->pkg->flags & PKGF_CORRUPT)) {
 477                                         ++stop_fail;
 478                                 }
 479                         }
 480                 }
 481         }
 482         printf("Total Returned %d\n", count);
 483
 484         /*
 485          * Check to see if any PKGF_MANUALSEL packages
 486          */
 487         if (stop_fail) {
 488                 printf("%d packages failed to probe\n", stop_fail);
 489                 if (PrepareSystemFlag) {
 490                         if (stop_fail == stop_base_list) {
 491                                 printf(
 492   "prepare-system: Some of your installed packages no longer exist in\n"
 493   "dports, do you wish to continue rebuilding what does exist?\n");
 494                                 if (askyn("Continue anyway? "))
 495                                         remove_corrupt = 1;
 496                         } else {
 497                                 printf(
 498   "prepare-system: Some of your installed packages have dependencies\n"
 499   "which could not be found in dports, cannot continue, aborting\n");
 500                         }
 501                 } else {
 502                         printf("unable to continue, aborting\n");
 503                 }
 504                 if (remove_corrupt == 0)
 505                         exit(1);
 506         }
 507
 508         /*
 509          * Remove corrupt packages before continuing
 510          */
 511         if (remove_corrupt) {
 512                 list_tail = &list;
 513                 while ((scan = *list_tail) != NULL) {
 514                         if (scan->flags & PKGF_CORRUPT)
 515                                 *list_tail = scan->bnext;
 516                         else
 517                                 list_tail = &scan->bnext;
 518                 }
 519         }
 520
 521         /*
 522          * Scan our binary distributions and related dependencies looking
 523          * for any packages that have already been built.
 524          */
 525         initbulk(childGetBinaryDistInfo, MaxBulk);
 526         total = scan_binary_repo(RepositoryPath);
 527         count = 0;
 528         printf("Scanning %d packages\n", total);
 529
 530         while ((bulk = getbulk()) != NULL) {
 531                 ++count;
 532                 if ((count & 255) == 0) {
 533                         printf("%6.2f%%\r",
 534                                 (double)count * 100.0 / (double)total + 0.001);
 535                         fflush(stdout);
 536                 }
 537                 freebulk(bulk);
 538         }
 539         printf("100.00%%\n");
 540         printf("\nTotal %d\n", count);
 541         fflush(stdout);
 542         donebulk();
 543
 544         printf("all done\n");
 545
 546         return list;
 547 }
 548
 549 pkg_t *
 550 GetPkgPkg(pkg_t **listp)
 551 {
 552         bulk_t *bulk;
 553         pkg_t *scan;
 554         pkg_t *s2;
 555
 556         for (scan = *listp; scan; scan = scan->bnext) {
 557                 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
 558                         return scan;
 559         }
 560
 561         /*
 562          * This will force pkg to be built, but generally this code
 563          * is not reached because the package list processing code
 564          * adds ports-mgmt/pkg unconditionally.
 565          */
 566         initbulk(childGetPackageInfo, MaxBulk);
 567         queuebulk("ports-mgmt", "pkg", NULL, "x");
 568         bulk = getbulk();
 569         dassert(bulk, "Cannot find ports-mgmt/pkg");
 570         scan = bulk->list;
 571         bulk->list = NULL;
 572         freebulk(bulk);
 573         donebulk();
 574
 575
 576         /*
 577          * Include added packages to the total and add the initial bulk
 578          * built packages to the list so they get counted.
 579          */
 580         for (s2 = scan; s2->bnext; s2 = s2->bnext)
 581                 ++BuildTotal;
 582         for (s2 = scan; s2->bnext; s2 = s2->bnext)
 583                 ;
 584         s2->bnext = *listp;
 585         *listp = scan;
 586         ++BuildTotal;
 587
 588         return scan;
 589 }
 590
 591 /*
 592  * Try to optimize the environment by supplying information that
 593  * the ports system would generally have to run stuff to get on
 594  * every package.
 595  *
 596  * See childOptimizeEnv() for the actual handling.  We execute
 597  * a single make -V... -V... for ports-mgmt/pkg from within the
 598  * bulk system (which handles the environment and disables
 599  * /etc/make.conf), and we then call addbuildenv() as appropriate.
 600  *
 601  * _PERL5_FROM_BIN
 602  * add others...
 603  */
 604 void
 605 OptimizeEnv(void)
 606 {
 607         bulk_t *bulk;
 608
 609         initbulk(childOptimizeEnv, MaxBulk);
 610         queuebulk("ports-mgmt", "pkg", NULL, NULL);
 611         bulk = getbulk();
 612         freebulk(bulk);
 613         donebulk();
 614 }
 615
 616 /*
 617  * Run through the list resolving dependencies and constructing the topology
 618  * linkages.   This may append packages to the list.  Dependencies to dummy
 619  * nodes which do not specify a flavor do not need special handling, the
 620  * search code in build.c will properly follow the first flavor.
 621  */
 622 static pkg_t *
 623 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
 624 {
 625         pkg_t *ret_list = NULL;
 626         pkg_t *scan;
 627         pkg_t *use;
 628         bulk_t *bulk;
 629
 630         for (scan = list; scan; scan = scan->bnext) {
 631                 use = pkg_find(scan->portdir);
 632                 resolveFlavors(use, scan->flavors, gentopo);
 633                 resolveDepString(use, scan->fetch_deps,
 634                                  gentopo, DEP_TYPE_FETCH);
 635                 resolveDepString(use, scan->ext_deps,
 636                                  gentopo, DEP_TYPE_EXT);
 637                 resolveDepString(use, scan->patch_deps,
 638                                  gentopo, DEP_TYPE_PATCH);
 639                 resolveDepString(use, scan->build_deps,
 640                                  gentopo, DEP_TYPE_BUILD);
 641                 resolveDepString(use, scan->lib_deps,
 642                                  gentopo, DEP_TYPE_LIB);
 643                 resolveDepString(use, scan->run_deps,
 644                                  gentopo, DEP_TYPE_RUN);
 645         }
 646
 647         /*
 648          * No bulk ops are queued when doing the final topology
 649          * generation.
 650          *
 651          * Avoid entering duplicate results from the bulk ops.  Duplicate
 652          * results are mostly filtered out, but not always.  A dummy node
 653          * representing multiple flavors will parse-out the flavors
 654          */
 655         if (gentopo)
 656                 return NULL;
 657         while ((bulk = getbulk()) != NULL) {
 658                 if (bulk->list) {
 659                         if (ret_list == NULL)
 660                                 ret_list = bulk->list;
 661                         **list_tailp = bulk->list;
 662                         bulk->list = NULL;
 663                         while ((scan = **list_tailp) != NULL) {
 664                                 pkg_enter(scan);
 665                                 *list_tailp = &scan->bnext;
 666                         }
 667                 }
 668                 freebulk(bulk);
 669         }
 670         return (ret_list);
 671 }
 672
 673 /*
 674  * Resolve a generic node that has flavors, queue to retrieve info for
 675  * each flavor and setup linkages as appropriate.
 676  */
 677 static void
 678 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo)
 679 {
 680         char *flavor_base;
 681         char *flavor_scan;
 682         char *flavor;
 683         char *portdir;
 684         char *s1;
 685         char *s2;
 686         pkg_t *dpkg;
 687         pkglink_t *link;
 688
 689         if ((pkg->flags & PKGF_DUMMY) == 0)
 690                 return;
 691         if (pkg->flavors == NULL || pkg->flavors[0] == 0)
 692                 return;
 693         flavor_base = strdup(flavors);
 694         flavor_scan = flavor_base;
 695
 696         for (;;) {
 697                 do {
 698                         flavor = strsep(&flavor_scan, " \t");
 699                 } while (flavor && *flavor == 0);
 700                 if (flavor == NULL)
 701                         break;
 702
 703                 /*
 704                  * Iterate each flavor generating "s1/s2@flavor".
 705                  *
 706                  * queuebulk() info for each flavor, and set-up the
 707                  * linkages in the topology generation pass.
 708                  */
 709                 asprintf(&portdir, "%s@%s", pkg->portdir, flavor);
 710                 s1 = strdup(pkg->portdir);
 711                 s2 = strchr(s1, '/');
 712                 *s2++ = 0;
 713
 714                 dpkg = pkg_find(portdir);
 715                 if (dpkg && gentopo) {
 716                         /*
 717                          * Setup linkages
 718                          */
 719                         free(portdir);
 720
 721                         link = calloc(1, sizeof(*link));
 722                         link->pkg = dpkg;
 723                         link->next = &pkg->idepon_list;
 724                         link->prev = pkg->idepon_list.prev;
 725                         link->next->prev = link;
 726                         link->prev->next = link;
 727                         link->dep_type = DEP_TYPE_BUILD;
 728
 729                         link = calloc(1, sizeof(*link));
 730                         link->pkg = pkg;
 731                         link->next = &dpkg->deponi_list;
 732                         link->prev = dpkg->deponi_list.prev;
 733                         link->next->prev = link;
 734                         link->prev->next = link;
 735                         link->dep_type = DEP_TYPE_BUILD;
 736                         ++dpkg->depi_count;
 737                 } else if (gentopo == 0 && dpkg == NULL) {
 738                         /*
 739                          * Use a place-holder to prevent duplicate
 740                          * dependencies from being processed.  The placeholder
 741                          * will be replaced by the actual dependency.
 742                          */
 743                         dpkg = allocpkg();
 744                         dpkg->portdir = portdir;
 745                         dpkg->flags = PKGF_PLACEHOLD;
 746                         pkg_enter(dpkg);
 747                         queuebulk(s1, s2, flavor, NULL);
 748                 }
 749                 free(s1);
 750         }
 751         free(flavor_base);
 752 }
 753
 754 static void
 755 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
 756 {
 757         char *copy_base;
 758         char *copy;
 759         char *dep;
 760         char *sep;
 761         char *tag;
 762         char *flavor;
 763         pkg_t *dpkg;
 764
 765         if (depstr == NULL || depstr[0] == 0)
 766                 return;
 767
 768         copy_base = strdup(depstr);
 769         copy = copy_base;
 770
 771         for (;;) {
 772                 do {
 773                         dep = strsep(&copy, " \t");
 774                 } while (dep && *dep == 0);
 775                 if (dep == NULL)
 776                         break;
 777
 778                 /*
 779                  * Ignore dependencies prefixed with ${NONEXISTENT}
 780                  */
 781                 if (strncmp(dep, "/nonexistent:", 13) == 0)
 782                         continue;
 783
 784                 dep = strchr(dep, ':');
 785                 if (dep == NULL || *dep != ':') {
 786                         printf("Error parsing dependency for %s: %s\n",
 787                                pkg->portdir, copy_base);
 788                         continue;
 789                 }
 790                 ++dep;
 791
 792                 /*
 793                  * Strip-off any DPortsPath prefix.  EXTRACT_DEPENDS
 794                  * often (always?) generates this prefix.
 795                  */
 796                 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
 797                         dep += strlen(DPortsPath);
 798                         if (*dep == '/')
 799                                 ++dep;
 800                 }
 801
 802                 /*
 803                  * Strip-off any tag (such as :patch).  We don't try to
 804                  * organize dependencies at this fine a grain (for now).
 805                  */
 806                 tag = strchr(dep, ':');
 807                 if (tag)
 808                         *tag++ = 0;
 809
 810                 /*
 811                  * Locate the dependency
 812                  */
 813                 if ((dpkg = pkg_find(dep)) != NULL) {
 814                         if (gentopo) {
 815                                 pkglink_t *link;
 816
 817                                 /*
 818                                  * NOTE: idep_count is calculated recursively
 819                                  *       at build-time
 820                                  */
 821                                 ddprintf(0, "Add Dependency %s -> %s\n",
 822                                         pkg->portdir, dpkg->portdir);
 823                                 link = calloc(1, sizeof(*link));
 824                                 link->pkg = dpkg;
 825                                 link->next = &pkg->idepon_list;
 826                                 link->prev = pkg->idepon_list.prev;
 827                                 link->next->prev = link;
 828                                 link->prev->next = link;
 829                                 link->dep_type = dep_type;
 830
 831                                 link = calloc(1, sizeof(*link));
 832                                 link->pkg = pkg;
 833                                 link->next = &dpkg->deponi_list;
 834                                 link->prev = dpkg->deponi_list.prev;
 835                                 link->next->prev = link;
 836                                 link->prev->next = link;
 837                                 link->dep_type = dep_type;
 838                                 ++dpkg->depi_count;
 839                         }
 840                         continue;
 841                 }
 842
 843                 /*
 844                  * This shouldn't happen because we already took a first
 845                  * pass and should have generated the pkgs.
 846                  */
 847                 if (gentopo) {
 848                         printf("Topology Generate failed for %s: %s\n",
 849                                 pkg->portdir, copy_base);
 850                         continue;
 851                 }
 852
 853                 /*
 854                  * Separate out the two dports directory components and
 855                  * extract the optional '@flavor' specification.
 856                  */
 857                 sep = strchr(dep, '/');
 858                 if (sep == NULL) {
 859                         printf("Error parsing dependency for %s: %s\n",
 860                                pkg->portdir, copy_base);
 861                         continue;
 862                 }
 863                 *sep++ = 0;
 864
 865                 /*
 866                  * The flavor hangs off the separator, not the tag
 867                  */
 868                 flavor = strrchr(sep, '@');
 869 #if 0
 870                 if (tag)
 871                         flavor = strrchr(tag, '@');
 872                 else
 873                         flavor = strrchr(sep, '@');
 874 #endif
 875                 if (flavor)
 876                         *flavor++ = 0;
 877
 878                 if (flavor)
 879                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
 880                                pkg->portdir, dep, sep, flavor);
 881                 else
 882                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
 883                                pkg->portdir, dep, sep);
 884
 885                 /*
 886                  * Use a place-holder to prevent duplicate dependencies from
 887                  * being processed.  The placeholder will be replaced by
 888                  * the actual dependency.
 889                  */
 890                 dpkg = allocpkg();
 891                 if (flavor)
 892                         asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
 893                 else
 894                         asprintf(&dpkg->portdir, "%s/%s", dep, sep);
 895                 dpkg->flags = PKGF_PLACEHOLD;
 896                 pkg_enter(dpkg);
 897
 898                 queuebulk(dep, sep, flavor, NULL);
 899         }
 900         free(copy_base);
 901 }
 902
 903 void
 904 FreePackageList(pkg_t *pkgs __unused)
 905 {
 906         dfatal("not implemented");
 907 }
 908
 909 /*
 910  * Scan some or all dports to allocate the related pkg structure.  Dependencies
 911  * are stored but not processed.
 912  *
 913  * Threaded function
 914  */
 915 static void
 916 childGetPackageInfo(bulk_t *bulk)
 917 {
 918         pkg_t *pkg;
 919         char *flavor;
 920         char *ptr;
 921         FILE *fp;
 922         int line;
 923         size_t len;
 924         char *portpath;
 925         char *flavarg;
 926         const char *cav[MAXCAC];
 927         pid_t pid;
 928         int cac;
 929
 930         /*
 931          * If the package has flavors we will loop on each one.  If a flavor
 932          * is not passed in s3 we will loop on all flavors, otherwise we will
 933          * only process the passed-in flavor.
 934          */
 935         flavor = bulk->s3;      /* usually NULL */
 936
 937         bulk->list = NULL;
 938
 939         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
 940         if (flavor)
 941                 asprintf(&flavarg, "FLAVOR=%s", flavor);
 942         else
 943                 flavarg = NULL;
 944
 945         cac = 0;
 946         cav[cac++] = MAKE_BINARY;
 947         cav[cac++] = "-C";
 948         cav[cac++] = portpath;
 949         if (flavarg)
 950                 cav[cac++] = flavarg;
 951         cav[cac++] = "-VPKGVERSION";
 952         cav[cac++] = "-VPKGFILE:T";
 953         cav[cac++] = "-VALLFILES";
 954         cav[cac++] = "-VDIST_SUBDIR";
 955         cav[cac++] = "-VMAKE_JOBS_NUMBER";
 956         cav[cac++] = "-VIGNORE";
 957         cav[cac++] = "-VFETCH_DEPENDS";
 958         cav[cac++] = "-VEXTRACT_DEPENDS";
 959         cav[cac++] = "-VPATCH_DEPENDS";
 960         cav[cac++] = "-VBUILD_DEPENDS";
 961         cav[cac++] = "-VLIB_DEPENDS";
 962         cav[cac++] = "-VRUN_DEPENDS";
 963         cav[cac++] = "-VSELECTED_OPTIONS";
 964         cav[cac++] = "-VDESELECTED_OPTIONS";
 965         cav[cac++] = "-VUSE_LINUX";
 966         cav[cac++] = "-VFLAVORS";
 967         cav[cac++] = "-VUSES";
 968
 969         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
 970         free(portpath);
 971         freestrp(&flavarg);
 972
 973         pkg = allocpkg();
 974         if (flavor)
 975                 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
 976         else
 977                 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
 978
 979         line = 1;
 980         while ((ptr = fgetln(fp, &len)) != NULL) {
 981                 if (len == 0 || ptr[len-1] != '\n') {
 982                         dfatal("Bad package info for %s/%s response line %d",
 983                                bulk->s1, bulk->s2, line);
 984                 }
 985                 ptr[--len] = 0;
 986
 987                 switch(line) {
 988                 case 1:         /* PKGVERSION */
 989                         asprintf(&pkg->version, "%s", ptr);
 990                         break;
 991                 case 2:         /* PKGFILE */
 992                         asprintf(&pkg->pkgfile, "%s", ptr);
 993                         break;
 994                 case 3:         /* ALLFILES (aka DISTFILES + patch files) */
 995                         asprintf(&pkg->distfiles, "%s", ptr);
 996                         break;
 997                 case 4:         /* DIST_SUBDIR */
 998                         pkg->distsubdir = strdup_or_null(ptr);
 999                         break;
1000                 case 5:         /* MAKE_JOBS_NUMBER */
1001                         pkg->make_jobs_number = strtol(ptr, NULL, 0);
1002                         break;
1003                 case 6:         /* IGNORE */
1004                         pkg->ignore = strdup_or_null(ptr);
1005                         break;
1006                 case 7:         /* FETCH_DEPENDS */
1007                         pkg->fetch_deps = strdup_or_null(ptr);
1008                         break;
1009                 case 8:         /* EXTRACT_DEPENDS */
1010                         pkg->ext_deps = strdup_or_null(ptr);
1011                         break;
1012                 case 9:         /* PATCH_DEPENDS */
1013                         pkg->patch_deps = strdup_or_null(ptr);
1014                         break;
1015                 case 10:        /* BUILD_DEPENDS */
1016                         pkg->build_deps = strdup_or_null(ptr);
1017                         break;
1018                 case 11:        /* LIB_DEPENDS */
1019                         pkg->lib_deps = strdup_or_null(ptr);
1020                         break;
1021                 case 12:        /* RUN_DEPENDS */
1022                         pkg->run_deps = strdup_or_null(ptr);
1023                         break;
1024                 case 13:        /* SELECTED_OPTIONS */
1025                         pkg->pos_options = strdup_or_null(ptr);
1026                         break;
1027                 case 14:        /* DESELECTED_OPTIONS */
1028                         pkg->neg_options = strdup_or_null(ptr);
1029                         break;
1030                 case 15:        /* USE_LINUX */
1031                         if (ptr[0])
1032                                 pkg->use_linux = 1;
1033                         break;
1034                 case 16:        /* FLAVORS */
1035                         asprintf(&pkg->flavors, "%s", ptr);
1036                         break;
1037                 case 17:        /* USES */
1038                         asprintf(&pkg->uses, "%s", ptr);
1039                         if (strstr(pkg->uses, "metaport"))
1040                                 pkg->flags |= PKGF_META;
1041                         break;
1042                 default:
1043                         printf("EXTRA LINE: %s\n", ptr);
1044                         break;
1045                 }
1046                 ++line;
1047         }
1048         if (line == 1) {
1049                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1050                 pkg->flags |= PKGF_NOTFOUND;
1051         } else if (line != 17 + 1) {
1052                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1053                 pkg->flags |= PKGF_CORRUPT;
1054         }
1055         if (dexec_close(fp, pid)) {
1056                 printf("make -V* command for %s/%s failed\n",
1057                         bulk->s1, bulk->s2);
1058                 pkg->flags |= PKGF_CORRUPT;
1059         }
1060         ddassert(bulk->s1);
1061
1062         /*
1063          * DEBUGSTOP mode
1064          */
1065         if (bulk->s4 && bulk->s4[0] == 'd')
1066                 pkg->flags |= PKGF_DEBUGSTOP;
1067
1068         /*
1069          * Mark as a dummy node, the front-end will iterate the flavors
1070          * and create sub-nodes for us.
1071          *
1072          * Get rid of elements returned that are for the first flavor.
1073          * We are creating a dummy node here, not the node for the first
1074          * flavor.
1075          */
1076         if (flavor == NULL && pkg->flavors && pkg->flavors[0]) {
1077                 pkg->flags |= PKGF_DUMMY;
1078                 freestrp(&pkg->fetch_deps);
1079                 freestrp(&pkg->ext_deps);
1080                 freestrp(&pkg->patch_deps);
1081                 freestrp(&pkg->build_deps);
1082                 freestrp(&pkg->lib_deps);
1083                 freestrp(&pkg->run_deps);
1084                 freestrp(&pkg->pkgfile);
1085         }
1086
1087         /*
1088          * Only one pkg is put on the return list now.  This code no
1089          * longer creates pseudo-nodes for flavors (the frontend requests
1090          * each flavor instead).
1091          */
1092         bulk->list = pkg;
1093 }
1094
1095 /*
1096  * Query the package (at least to make sure it hasn't been truncated)
1097  * and mark it as PACKAGED if found.
1098  *
1099  * Threaded function
1100  */
1101 static void
1102 childGetBinaryDistInfo(bulk_t *bulk)
1103 {
1104         char *ptr;
1105         FILE *fp;
1106         size_t len;
1107         pkg_t *pkg;
1108         const char *cav[MAXCAC];
1109         char *repopath;
1110         char buf[1024];
1111         pid_t pid;
1112         int cac;
1113         int deleteme;
1114
1115         asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
1116
1117         cac = 0;
1118         cav[cac++] = PKG_BINARY;
1119         cav[cac++] = "query";
1120         cav[cac++] = "-F";
1121         cav[cac++] = repopath;
1122         cav[cac++] = "%n-%v";
1123
1124         fp = dexec_open(cav, cac, &pid, NULL, 1, 0);
1125         deleteme = DeleteObsoletePkgs;
1126
1127         while ((ptr = fgetln(fp, &len)) != NULL) {
1128                 if (len == 0 || ptr[len-1] != '\n')
1129                         continue;
1130                 ptr[len-1] = 0;
1131                 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
1132
1133                 pkg = pkg_find(buf);
1134                 if (pkg) {
1135                         pkg->flags |= PKGF_PACKAGED;
1136                         deleteme = 0;
1137                 } else {
1138                         ddprintf(0, "Note: package scan, not in list, "
1139                                     "skipping %s\n", buf);
1140                 }
1141         }
1142         if (dexec_close(fp, pid)) {
1143                 printf("pkg query command failed for %s\n", repopath);
1144         }
1145         if (deleteme) {
1146                 dlog(DLOG_ALL | DLOG_STDOUT,
1147                      "Deleting obsolete package %s\n", repopath);
1148                 remove(repopath);
1149         }
1150         free(repopath);
1151 }
1152
1153 static void
1154 childOptimizeEnv(bulk_t *bulk)
1155 {
1156         char *portpath;
1157         char *ptr;
1158         FILE *fp;
1159         int line;
1160         size_t len;
1161         const char *cav[MAXCAC];
1162         pid_t pid;
1163         int cac;
1164
1165         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
1166
1167         cac = 0;
1168         cav[cac++] = MAKE_BINARY;
1169         cav[cac++] = "-C";
1170         cav[cac++] = portpath;
1171         cav[cac++] = "-V_PERL5_FROM_BIN";
1172
1173         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
1174         free(portpath);
1175
1176         line = 1;
1177         while ((ptr = fgetln(fp, &len)) != NULL) {
1178                 if (len == 0 || ptr[len-1] != '\n') {
1179                         dfatal("Bad package info for %s/%s response line %d",
1180                                bulk->s1, bulk->s2, line);
1181                 }
1182                 ptr[--len] = 0;
1183
1184                 switch(line) {
1185                 case 1:         /* _PERL5_FROM_BIN */
1186                         addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1187                         break;
1188                 default:
1189                         printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1190                         break;
1191                 }
1192                 ++line;
1193         }
1194         if (line == 1) {
1195                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1196         } else if (line != 1 + 1) {
1197                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1198         }
1199         if (dexec_close(fp, pid)) {
1200                 printf("childOptimizeEnv() failed\n");
1201         }
1202 }
1203
1204 static int
1205 scan_and_queue_dir(const char *path, const char *level1, int level)
1206 {
1207         DIR *dir;
1208         char *s1;
1209         char *s2;
1210         struct dirent *den;
1211         struct stat st;
1212         int count = 0;
1213
1214         dir = opendir(path);
1215         dassert(dir, "Cannot open dports path \"%s\"", path);
1216
1217         while ((den = readdir(dir)) != NULL) {
1218                 if (den->d_namlen == 1 && den->d_name[0] == '.')
1219                         continue;
1220                 if (den->d_namlen == 2 &&
1221                     den->d_name[0] == '.' && den->d_name[1] == '.')
1222                         continue;
1223                 asprintf(&s1, "%s/%s", path, den->d_name);
1224                 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1225                         free(s1);
1226                         continue;
1227                 }
1228                 if (level == 1) {
1229                         count += scan_and_queue_dir(s1, den->d_name, 2);
1230                         free(s1);
1231                         continue;
1232                 }
1233                 asprintf(&s2, "%s/Makefile", s1);
1234                 if (lstat(s2, &st) == 0) {
1235                         queuebulk(level1, den->d_name, NULL, NULL);
1236                         ++count;
1237                 }
1238                 free(s1);
1239                 free(s2);
1240         }
1241         closedir(dir);
1242
1243         return count;
1244 }
1245
1246 static int
1247 scan_binary_repo(const char *path)
1248 {
1249         DIR *dir;
1250         struct dirent *den;
1251         size_t len;
1252         int count;
1253
1254         count = 0;
1255         dir = opendir(path);
1256         dassert(dir, "Cannot open repository path \"%s\"", path);
1257
1258         /*
1259          * NOTE: Test includes the '.' in the suffix.
1260          */
1261         while ((den = readdir(dir)) != NULL) {
1262                 len = strlen(den->d_name);
1263                 if (len > 4 &&
1264                     strcmp(den->d_name + len - 4, UsePkgSufx) == 0) {
1265                         queuebulk(den->d_name, NULL, NULL, NULL);
1266                         ++count;
1267                 }
1268         }
1269         closedir(dir);
1270
1271         return count;
1272 }
1273
1274 #if 0
1275 static void
1276 pkgfree(pkg_t *pkg)
1277 {
1278         freestrp(&pkg->portdir);
1279         freestrp(&pkg->version);
1280         freestrp(&pkg->pkgfile);
1281         freestrp(&pkg->ignore);
1282         freestrp(&pkg->fetch_deps);
1283         freestrp(&pkg->ext_deps);
1284         freestrp(&pkg->patch_deps);
1285         freestrp(&pkg->build_deps);
1286         freestrp(&pkg->lib_deps);
1287         freestrp(&pkg->run_deps);
1288         freestrp(&pkg->pos_options);
1289         freestrp(&pkg->neg_options);
1290         freestrp(&pkg->flavors);
1291         free(pkg);
1292 }
1293 #endif