usr.bin/dsynth/pkglist.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37
  38 #include "dsynth.h"
  39
  40 #define PKG_HSIZE       32768
  41 #define PKG_HMASK       32767
  42
  43 static int parsepkglist_file(const char *path, int debugstop);
  44 static void childGetPackageInfo(bulk_t *bulk);
  45 static void childGetBinaryDistInfo(bulk_t *bulk);
  46 static void childOptimizeEnv(bulk_t *bulk);
  47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
  48 static void resolveDepString(pkg_t *pkg, char *depstr,
  49                         int gentopo, int dep_type);
  50 static pkg_t *processPackageListBulk(int total);
  51 static int scan_and_queue_dir(const char *path, const char *level1, int level);
  52 static int scan_binary_repo(const char *path);
  53 #if 0
  54 static void pkgfree(pkg_t *pkg);
  55 #endif
  56
  57 pkg_t *PkgHash1[PKG_HSIZE];     /* by portdir */
  58 pkg_t *PkgHash2[PKG_HSIZE];     /* by pkgfile */
  59
  60 /*
  61  * Allocate a new pkg structure plus basic initialization.
  62  */
  63 static __inline pkg_t *
  64 allocpkg(void)
  65 {
  66         pkg_t *pkg;
  67
  68         pkg = calloc(1, sizeof(*pkg));
  69         pkg->idepon_list.next = &pkg->idepon_list;
  70         pkg->idepon_list.prev = &pkg->idepon_list;
  71         pkg->deponi_list.next = &pkg->deponi_list;
  72         pkg->deponi_list.prev = &pkg->deponi_list;
  73
  74         return pkg;
  75 }
  76
  77 /*
  78  * Simple hash for lookups
  79  */
  80 static __inline int
  81 pkghash(const char *str)
  82 {
  83         int hv = 0xABC32923;
  84         while (*str) {
  85                 hv = (hv << 5) ^ *str;
  86                 ++str;
  87         }
  88         hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
  89         return (hv & PKG_HMASK);
  90 }
  91
  92 static void
  93 pkg_enter(pkg_t *pkg)
  94 {
  95         pkg_t **pkgp;
  96         pkg_t *scan;
  97
  98         if (pkg->portdir) {
  99                 pkgp = &PkgHash1[pkghash(pkg->portdir)];
 100                 while ((scan = *pkgp) != NULL) {
 101                         if (strcmp(pkg->portdir, scan->portdir) == 0)
 102                                 break;
 103                         pkgp = &scan->hnext1;
 104                 }
 105                 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
 106                         *pkgp = pkg;
 107                         pkg->hnext1 = scan->hnext1;
 108                         free(scan->portdir);
 109                         free(scan);
 110                         scan = NULL;
 111                 }
 112                 if (scan == NULL)
 113                         *pkgp = pkg;
 114         }
 115
 116         if (pkg->pkgfile) {
 117                 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
 118                 while ((scan = *pkgp) != NULL) {
 119                         if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
 120                                 break;
 121                         pkgp = &scan->hnext2;
 122                 }
 123                 if (scan == NULL)
 124                         *pkgp = pkg;
 125         }
 126 }
 127
 128 static pkg_t *
 129 pkg_find(const char *match)
 130 {
 131         pkg_t **pkgp;
 132         pkg_t *pkg;
 133
 134         pkgp = &PkgHash1[pkghash(match)];
 135         for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
 136                 if (strcmp(pkg->portdir, match) == 0)
 137                         return pkg;
 138         }
 139         pkgp = &PkgHash2[pkghash(match)];
 140         for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
 141                 if (strcmp(pkg->pkgfile, match) == 0)
 142                         return pkg;
 143         }
 144         return NULL;
 145 }
 146
 147 /*
 148  * Parse a specific list of ports via origin name (portdir/subdir)
 149  */
 150 pkg_t *
 151 ParsePackageList(int n, char **ary, int debugstop)
 152 {
 153         pkg_t *list;
 154         int i;
 155         int total;
 156
 157         total = 0;
 158         initbulk(childGetPackageInfo, MaxBulk);
 159
 160         /*
 161          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 162          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 163          */
 164         queuebulk("ports-mgmt", "pkg", NULL, "x");
 165
 166         for (i = 0; i < n; ++i) {
 167                 char *l1;
 168                 char *l2;
 169                 char *l3;
 170                 struct stat st;
 171
 172                 l1 = strdup(ary[i]);
 173                 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
 174                         total += parsepkglist_file(l1, debugstop);
 175                         continue;
 176                 }
 177
 178                 l2 = strchr(l1, '/');
 179                 if (l2 == NULL) {
 180                         printf("Bad portdir specification: %s\n", l1);
 181                         free(l1);
 182                         continue;
 183                 }
 184                 *l2++ = 0;
 185                 l3 = strchr(l2, '@');
 186                 if (l3)
 187                         *l3++ = 0;
 188                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 189                 ++total;
 190                 free(l1);
 191         }
 192         printf("Processing %d ports\n", total);
 193
 194         list = processPackageListBulk(total);
 195
 196         return list;
 197 }
 198
 199 static
 200 int
 201 parsepkglist_file(const char *path, int debugstop)
 202 {
 203         FILE *fp;
 204         char *base;
 205         char *l1;
 206         char *l2;
 207         char *l3;
 208         size_t len;
 209         int total;
 210
 211         if ((fp = fopen(path, "r")) == NULL) {
 212                 dpanic_errno("Cannot read %s\n", path);
 213                 /* NOT REACHED */
 214                 return 0;
 215         }
 216
 217         total = 0;
 218
 219         while ((base = fgetln(fp, &len)) != NULL) {
 220                 if (len == 0 || base[len-1] != '\n')
 221                         continue;
 222                 base[--len] = 0;
 223                 l1 = strtok(base, " \t\r\n");
 224                 if (l1 == NULL) {
 225                         printf("Badly formatted pkg info line: %s\n", base);
 226                         continue;
 227                 }
 228                 l2 = strchr(l1, '/');
 229                 if (l2 == NULL) {
 230                         printf("Badly formatted specification: %s\n", l1);
 231                         continue;
 232                 }
 233                 *l2++ = 0;
 234                 l3 = strchr(l2, '@');
 235                 if (l3)
 236                         *l3++ = 0;
 237                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 238                 ++total;
 239         }
 240         fclose(fp);
 241
 242         return total;
 243 }
 244
 245 /*
 246  * Parse packages from the list installed on the system.
 247  */
 248 pkg_t *
 249 GetLocalPackageList(void)
 250 {
 251         pkg_t *list;
 252         FILE *fp;
 253         char *base;
 254         char *l1;
 255         char *l2;
 256         char *l3;
 257         int total;
 258         size_t len;
 259
 260         initbulk(childGetPackageInfo, MaxBulk);
 261         total = 0;
 262
 263         fp = popen("pkg info -a -o", "r");
 264
 265         /*
 266          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 267          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 268          */
 269         queuebulk("ports-mgmt", "pkg", NULL, "x");
 270
 271         while ((base = fgetln(fp, &len)) != NULL) {
 272                 if (len == 0 || base[len-1] != '\n')
 273                         continue;
 274                 base[--len] = 0;
 275                 if (strtok(base, " \t") == NULL) {
 276                         printf("Badly formatted pkg info line: %s\n", base);
 277                         continue;
 278                 }
 279                 l1 = strtok(NULL, " \t");
 280                 if (l1 == NULL) {
 281                         printf("Badly formatted pkg info line: %s\n", base);
 282                         continue;
 283                 }
 284
 285                 l2 = strchr(l1, '/');
 286                 if (l2 == NULL) {
 287                         printf("Badly formatted specification: %s\n", l1);
 288                         continue;
 289                 }
 290                 *l2++ = 0;
 291                 l3 = strchr(l2, '@');
 292                 if (l3)
 293                         *l3++ = 0;
 294                 queuebulk(l1, l2, l3, NULL);
 295                 ++total;
 296         }
 297         pclose(fp);
 298
 299         printf("Processing %d ports\n", total);
 300
 301         list = processPackageListBulk(total);
 302
 303         return list;
 304 }
 305
 306 pkg_t *
 307 GetFullPackageList(void)
 308 {
 309         int total;
 310
 311         initbulk(childGetPackageInfo, MaxBulk);
 312
 313         total = scan_and_queue_dir(DPortsPath, NULL, 1);
 314         printf("Scanning %d ports\n", total);
 315
 316         return processPackageListBulk(total);
 317 }
 318
 319 /*
 320  * Caller has queued the process list for bulk operation.  We retrieve
 321  * the results and clean up the bulk operation (we may have to do a second
 322  * bulk operation so we have to be the ones to clean it up).
 323  */
 324 static pkg_t *
 325 processPackageListBulk(int total)
 326 {
 327         bulk_t *bulk;
 328         pkg_t *scan;
 329         pkg_t *list;
 330         pkg_t *dep_list;
 331         pkg_t **list_tail;
 332         int count;
 333
 334         list = NULL;
 335         list_tail = &list;
 336         count = 0;
 337
 338         while ((bulk = getbulk()) != NULL) {
 339                 ++count;
 340                 if ((count & 255) == 0) {
 341                         printf("%6.2f%%\r",
 342                                 (double)count * 100.0 / (double)total + 0.001);
 343                         fflush(stdout);
 344                 }
 345                 if (bulk->list) {
 346                         *list_tail = bulk->list;
 347                         bulk->list = NULL;
 348                         while ((scan = *list_tail) != NULL) {
 349                                 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
 350                                         scan->flags |= PKGF_MANUALSEL;
 351                                 pkg_enter(scan);
 352                                 list_tail = &scan->bnext;
 353                         }
 354                 }
 355                 freebulk(bulk);
 356         }
 357         printf("100.00%%\n");
 358         printf("\nTotal %d\n", count);
 359         fflush(stdout);
 360
 361         /*
 362          * Resolve all dependencies for the related packages, potentially
 363          * adding anything that could not be found to the list.  This will
 364          * continue to issue bulk operations and process the result until
 365          * no dependencies are left.
 366          */
 367         printf("Resolving dependencies...");
 368         fflush(stdout);
 369         dep_list = list;
 370         while (dep_list) {
 371                 dep_list = resolveDeps(dep_list, &list_tail, 0);
 372         }
 373         printf("done\n");
 374
 375         donebulk();
 376
 377         /*
 378          * Generate the topology
 379          */
 380         resolveDeps(list, NULL, 1);
 381
 382         /*
 383          * Do a final count, ignore place holders.
 384          */
 385         count = 0;
 386         for (scan = list; scan; scan = scan->bnext) {
 387                 if ((scan->flags & PKGF_ERROR) == 0) {
 388                         ++count;
 389                 }
 390         }
 391         printf("Total Returned %d\n", count);
 392
 393         /*
 394          * Scan our binary distributions and related dependencies looking
 395          * for any packages that have already been built.
 396          */
 397         initbulk(childGetBinaryDistInfo, MaxBulk);
 398         total = scan_binary_repo(RepositoryPath);
 399         count = 0;
 400         printf("Scanning %d packages\n", total);
 401
 402         while ((bulk = getbulk()) != NULL) {
 403                 ++count;
 404                 if ((count & 255) == 0) {
 405                         printf("%6.2f%%\r",
 406                                 (double)count * 100.0 / (double)total + 0.001);
 407                         fflush(stdout);
 408                 }
 409                 freebulk(bulk);
 410         }
 411         printf("100.00%%\n");
 412         printf("\nTotal %d\n", count);
 413         fflush(stdout);
 414         donebulk();
 415
 416         printf("all done\n");
 417
 418         return list;
 419 }
 420
 421 pkg_t *
 422 GetPkgPkg(pkg_t *list)
 423 {
 424         bulk_t *bulk;
 425         pkg_t *scan;
 426
 427         for (scan = list; scan; scan = scan->bnext) {
 428                 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
 429                         return scan;
 430         }
 431
 432         /*
 433          * This will force pkg to be built, but generally this code
 434          * is not reached because the package list processing code
 435          * adds ports-mgmt/pkg unconditionally.
 436          */
 437         initbulk(childGetPackageInfo, MaxBulk);
 438         queuebulk("ports-mgmt", "pkg", NULL, "x");
 439         bulk = getbulk();
 440         dassert(bulk, "Cannot find ports-mgmt/pkg");
 441         scan = bulk->list;
 442         bulk->list = NULL;
 443         freebulk(bulk);
 444         donebulk();
 445
 446         return scan;
 447 }
 448
 449 /*
 450  * Try to optimize the environment by supplying information that
 451  * the ports system would generally have to run stuff to get on
 452  * every package.
 453  *
 454  * See childOptimizeEnv() for the actual handling.  We execute
 455  * a single make -V... -V... for ports-mgmt/pkg from within the
 456  * bulk system (which handles the environment and disables
 457  * /etc/make.conf), and we then call addbuildenv() as appropriate.
 458  *
 459  * _PERL5_FROM_BIN
 460  * add others...
 461  */
 462 void
 463 OptimizeEnv(void)
 464 {
 465         bulk_t *bulk;
 466
 467         initbulk(childOptimizeEnv, MaxBulk);
 468         queuebulk("ports-mgmt", "pkg", NULL, NULL);
 469         bulk = getbulk();
 470         freebulk(bulk);
 471         donebulk();
 472 }
 473
 474 /*
 475  * Run through the list resolving dependencies and constructing the topology
 476  * linkages.   This may append packages to the list.
 477  */
 478 static pkg_t *
 479 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
 480 {
 481         pkg_t *scan;
 482         pkg_t *ret_list = NULL;
 483         bulk_t *bulk;
 484
 485         for (scan = list; scan; scan = scan->bnext) {
 486                 resolveDepString(scan, scan->fetch_deps,
 487                                  gentopo, DEP_TYPE_FETCH);
 488                 resolveDepString(scan, scan->ext_deps,
 489                                  gentopo, DEP_TYPE_EXT);
 490                 resolveDepString(scan, scan->patch_deps,
 491                                  gentopo, DEP_TYPE_PATCH);
 492                 resolveDepString(scan, scan->build_deps,
 493                                  gentopo, DEP_TYPE_BUILD);
 494                 resolveDepString(scan, scan->lib_deps,
 495                                  gentopo, DEP_TYPE_LIB);
 496                 resolveDepString(scan, scan->run_deps,
 497                                  gentopo, DEP_TYPE_RUN);
 498         }
 499
 500         /*
 501          * No bulk ops are queued when doing the final topology
 502          * generation.
 503          */
 504         if (gentopo)
 505                 return NULL;
 506         while ((bulk = getbulk()) != NULL) {
 507                 if (bulk->list) {
 508                         if (ret_list == NULL)
 509                                 ret_list = bulk->list;
 510                         **list_tailp = bulk->list;
 511                         bulk->list = NULL;
 512                         while (**list_tailp) {
 513                                 pkg_enter(**list_tailp);
 514                                 *list_tailp = &(**list_tailp)->bnext;
 515                         }
 516                 }
 517                 freebulk(bulk);
 518         }
 519         return (ret_list);
 520 }
 521
 522 static void
 523 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
 524 {
 525         char *copy_base;
 526         char *copy;
 527         char *dep;
 528         char *sep;
 529         char *tag;
 530         char *flavor;
 531         pkg_t *dpkg;
 532
 533         if (depstr == NULL || depstr[0] == 0)
 534                 return;
 535
 536         copy_base = strdup(depstr);
 537         copy = copy_base;
 538
 539         for (;;) {
 540                 do {
 541                         dep = strsep(&copy, " \t");
 542                 } while (dep && *dep == 0);
 543                 if (dep == NULL)
 544                         break;
 545
 546                 /*
 547                  * Ignore dependencies prefixed with ${NONEXISTENT}
 548                  */
 549                 if (strncmp(dep, "/nonexistent:", 13) == 0)
 550                         continue;
 551
 552                 dep = strchr(dep, ':');
 553                 if (dep == NULL || *dep != ':') {
 554                         printf("Error parsing dependency for %s: %s\n",
 555                                pkg->portdir, copy_base);
 556                         continue;
 557                 }
 558                 ++dep;
 559
 560                 /*
 561                  * Strip-off any DPortsPath prefix.  EXTRACT_DEPENDS
 562                  * often (always?) generates this prefix.
 563                  */
 564                 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
 565                         dep += strlen(DPortsPath);
 566                         if (*dep == '/')
 567                                 ++dep;
 568                 }
 569
 570                 /*
 571                  * Strip-off any tag (such as :patch).  We don't try to
 572                  * organize dependencies at this fine a grain (for now).
 573                  */
 574                 tag = strchr(dep, ':');
 575                 if (tag)
 576                         *tag++ = 0;
 577
 578                 /*
 579                  * Locate the dependency
 580                  */
 581                 if ((dpkg = pkg_find(dep)) != NULL) {
 582                         if (gentopo) {
 583                                 pkglink_t *link;
 584
 585                                 /*
 586                                  * NOTE: idep_count is calculated recursively
 587                                  *       at build-time
 588                                  */
 589                                 ddprintf(0, "Add Dependency %s -> %s\n",
 590                                         pkg->portdir, dpkg->portdir);
 591                                 link = calloc(1, sizeof(*link));
 592                                 link->pkg = dpkg;
 593                                 link->next = &pkg->idepon_list;
 594                                 link->prev = pkg->idepon_list.prev;
 595                                 link->next->prev = link;
 596                                 link->prev->next = link;
 597                                 link->dep_type = dep_type;
 598
 599                                 link = calloc(1, sizeof(*link));
 600                                 link->pkg = pkg;
 601                                 link->next = &dpkg->deponi_list;
 602                                 link->prev = dpkg->deponi_list.prev;
 603                                 link->next->prev = link;
 604                                 link->prev->next = link;
 605                                 link->dep_type = dep_type;
 606                                 ++dpkg->depi_count;
 607                         }
 608                         continue;
 609                 }
 610
 611                 /*
 612                  * This shouldn't happen because we already took a first
 613                  * pass and should have generated the pkgs.
 614                  */
 615                 if (gentopo) {
 616                         printf("Topology Generate failed for %s: %s\n",
 617                                 pkg->portdir, copy_base);
 618                         continue;
 619                 }
 620
 621                 /*
 622                  * Separate out the two dports directory components and
 623                  * extract the optional '@flavor' specification.
 624                  */
 625                 sep = strchr(dep, '/');
 626                 if (sep == NULL) {
 627                         printf("Error parsing dependency for %s: %s\n",
 628                                pkg->portdir, copy_base);
 629                         continue;
 630                 }
 631                 *sep++ = 0;
 632
 633                 if (tag)
 634                         flavor = strrchr(tag, '@');
 635                 else
 636                         flavor = strrchr(sep, '@');
 637
 638                 if (flavor)
 639                         *flavor++ = 0;
 640
 641                 if (flavor)
 642                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
 643                                pkg->portdir, dep, sep, flavor);
 644                 else
 645                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
 646                                pkg->portdir, dep, sep);
 647
 648                 /*
 649                  * Use a place-holder to prevent duplicate dependencies from
 650                  * being processed.  The placeholder will be replaced by
 651                  * the actual dependency.
 652                  */
 653                 dpkg = allocpkg();
 654                 if (flavor)
 655                         asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
 656                 else
 657                         asprintf(&dpkg->portdir, "%s/%s", dep, sep);
 658                 dpkg->flags = PKGF_PLACEHOLD;
 659                 pkg_enter(dpkg);
 660
 661                 queuebulk(dep, sep, flavor, NULL);
 662         }
 663         free(copy_base);
 664 }
 665
 666 void
 667 FreePackageList(pkg_t *pkgs __unused)
 668 {
 669         dfatal("not implemented");
 670 }
 671
 672 /*
 673  * Scan some or all dports to allocate the related pkg structure.  Dependencies
 674  * are stored but not processed.
 675  *
 676  * Threaded function
 677  */
 678 static void
 679 childGetPackageInfo(bulk_t *bulk)
 680 {
 681         pkg_t *pkg;
 682         pkg_t *dummy_node;
 683         pkg_t **list_tail;
 684         char *flavors_save;
 685         char *flavors;
 686         char *flavor;
 687         char *ptr;
 688         FILE *fp;
 689         int line;
 690         size_t len;
 691         char *portpath;
 692         char *flavarg;
 693         const char *cav[MAXCAC];
 694         pid_t pid;
 695         int cac;
 696
 697         /*
 698          * If the package has flavors we will loop on each one.  If a flavor
 699          * is not passed in s3 we will loop on all flavors, otherwise we will
 700          * only process the passed-in flavor.
 701          */
 702         flavor = bulk->s3;      /* usually NULL */
 703         flavors = NULL;
 704         flavors_save = NULL;
 705         dummy_node = NULL;
 706
 707         bulk->list = NULL;
 708         list_tail = &bulk->list;
 709 again:
 710         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
 711         if (flavor)
 712                 asprintf(&flavarg, "FLAVOR=%s", flavor);
 713         else
 714                 flavarg = NULL;
 715
 716         cac = 0;
 717         cav[cac++] = MAKE_BINARY;
 718         cav[cac++] = "-C";
 719         cav[cac++] = portpath;
 720         if (flavarg)
 721                 cav[cac++] = flavarg;
 722         cav[cac++] = "-VPKGVERSION";
 723         cav[cac++] = "-VPKGFILE:T";
 724         cav[cac++] = "-VDISTFILES";
 725         cav[cac++] = "-VDIST_SUBDIR";
 726         cav[cac++] = "-VMAKE_JOBS_NUMBER";
 727         cav[cac++] = "-VIGNORE";
 728         cav[cac++] = "-VFETCH_DEPENDS";
 729         cav[cac++] = "-VEXTRACT_DEPENDS";
 730         cav[cac++] = "-VPATCH_DEPENDS";
 731         cav[cac++] = "-VBUILD_DEPENDS";
 732         cav[cac++] = "-VLIB_DEPENDS";
 733         cav[cac++] = "-VRUN_DEPENDS";
 734         cav[cac++] = "-VSELECTED_OPTIONS";
 735         cav[cac++] = "-VDESELECTED_OPTIONS";
 736         cav[cac++] = "-VUSE_LINUX";
 737         cav[cac++] = "-VFLAVORS";
 738         cav[cac++] = "-VUSES";
 739
 740         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
 741         free(portpath);
 742         freestrp(&flavarg);
 743
 744         pkg = allocpkg();
 745         if (flavor)
 746                 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
 747         else
 748                 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
 749
 750         line = 1;
 751         while ((ptr = fgetln(fp, &len)) != NULL) {
 752                 if (len == 0 || ptr[len-1] != '\n') {
 753                         dfatal("Bad package info for %s/%s response line %d",
 754                                bulk->s1, bulk->s2, line);
 755                 }
 756                 ptr[--len] = 0;
 757
 758                 switch(line) {
 759                 case 1:         /* PKGVERSION */
 760                         asprintf(&pkg->version, "%s", ptr);
 761                         break;
 762                 case 2:         /* PKGFILE */
 763                         asprintf(&pkg->pkgfile, "%s", ptr);
 764                         break;
 765                 case 3:         /* DISTFILES */
 766                         asprintf(&pkg->distfiles, "%s", ptr);
 767                         break;
 768                 case 4:         /* DIST_SUBDIR */
 769                         pkg->distsubdir = strdup_or_null(ptr);
 770                         break;
 771                 case 5:         /* MAKE_JOBS_NUMBER */
 772                         pkg->make_jobs_number = strtol(ptr, NULL, 0);
 773                         break;
 774                 case 6:         /* IGNORE */
 775                         pkg->ignore = strdup_or_null(ptr);
 776                         break;
 777                 case 7:         /* FETCH_DEPENDS */
 778                         pkg->fetch_deps = strdup_or_null(ptr);
 779                         break;
 780                 case 8:         /* EXTRACT_DEPENDS */
 781                         pkg->ext_deps = strdup_or_null(ptr);
 782                         break;
 783                 case 9:         /* PATCH_DEPENDS */
 784                         pkg->patch_deps = strdup_or_null(ptr);
 785                         break;
 786                 case 10:        /* BUILD_DEPENDS */
 787                         pkg->build_deps = strdup_or_null(ptr);
 788                         break;
 789                 case 11:        /* LIB_DEPENDS */
 790                         pkg->lib_deps = strdup_or_null(ptr);
 791                         break;
 792                 case 12:        /* RUN_DEPENDS */
 793                         pkg->run_deps = strdup_or_null(ptr);
 794                         break;
 795                 case 13:        /* SELECTED_OPTIONS */
 796                         pkg->pos_options = strdup_or_null(ptr);
 797                         break;
 798                 case 14:        /* DESELECTED_OPTIONS */
 799                         pkg->neg_options = strdup_or_null(ptr);
 800                         break;
 801                 case 15:        /* USE_LINUX */
 802                         if (ptr[0])
 803                                 pkg->use_linux = 1;
 804                         break;
 805                 case 16:        /* FLAVORS */
 806                         asprintf(&pkg->flavors, "%s", ptr);
 807                         break;
 808                 case 17:        /* USES */
 809                         asprintf(&pkg->uses, "%s", ptr);
 810                         if (strstr(pkg->uses, "metaport"))
 811                                 pkg->flags |= PKGF_META;
 812                         break;
 813                 default:
 814                         printf("EXTRA LINE: %s\n", ptr);
 815                         break;
 816                 }
 817                 ++line;
 818         }
 819         if (line == 1) {
 820                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
 821                 pkg->flags |= PKGF_NOTFOUND;
 822         } else if (line != 17 + 1) {
 823                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
 824                 pkg->flags |= PKGF_CORRUPT;
 825         }
 826         if (dexec_close(fp, pid)) {
 827                 printf("make -V* command for %s/%s failed\n",
 828                         bulk->s1, bulk->s2);
 829                 pkg->flags |= PKGF_CORRUPT;
 830         }
 831         ddassert(bulk->s1);
 832
 833         /*
 834          * DEBUGSTOP mode
 835          */
 836         if (bulk->s4 && bulk->s4[0] == 'd')
 837                 pkg->flags |= PKGF_DEBUGSTOP;
 838
 839         /*
 840          * Generate flavors
 841          */
 842         if (flavor == NULL) {
 843                 /*
 844                  * If there are flavors add the current unflavored pkg
 845                  * as a dummy node so dependencies can attach to it,
 846                  * then iterate the first flavor and loop.
 847                  *
 848                  * We must NULL out pkgfile because it will have the
 849                  * default flavor and conflict with the actual flavored
 850                  * pkg.
 851                  */
 852                 if (pkg->flavors && pkg->flavors[0]) {
 853                         dummy_node = pkg;
 854
 855                         pkg->flags |= PKGF_DUMMY;
 856
 857                         freestrp(&pkg->fetch_deps);
 858                         freestrp(&pkg->ext_deps);
 859                         freestrp(&pkg->patch_deps);
 860                         freestrp(&pkg->build_deps);
 861                         freestrp(&pkg->lib_deps);
 862                         freestrp(&pkg->run_deps);
 863
 864                         freestrp(&pkg->pkgfile);
 865                         *list_tail = pkg;
 866                         while (*list_tail)
 867                                 list_tail = &(*list_tail)->bnext;
 868
 869                         flavors_save = strdup(pkg->flavors);
 870                         flavors = flavors_save;
 871                         do {
 872                                 flavor = strsep(&flavors, " \t");
 873                         } while (flavor && *flavor == 0);
 874                         goto again;
 875                 }
 876
 877                 /*
 878                  * No flavors, add the current unflavored pkg as a real
 879                  * node.
 880                  */
 881                 *list_tail = pkg;
 882                 while (*list_tail)
 883                         list_tail = &(*list_tail)->bnext;
 884         } else {
 885                 /*
 886                  * Add flavored package and iterate.
 887                  */
 888                 *list_tail = pkg;
 889                 while (*list_tail)
 890                         list_tail = &(*list_tail)->bnext;
 891
 892                 /*
 893                  * Flavor iteration under dummy node, add dependency
 894                  */
 895                 if (dummy_node) {
 896                         pkglink_t *link;
 897
 898                         ddprintf(0, "Add Dependency %s -> %s (flavor rollup)\n",
 899                                 dummy_node->portdir, pkg->portdir);
 900                         link = calloc(1, sizeof(*link));
 901                         link->pkg = pkg;
 902                         link->next = &dummy_node->idepon_list;
 903                         link->prev = dummy_node->idepon_list.prev;
 904                         link->next->prev = link;
 905                         link->prev->next = link;
 906                         link->dep_type = DEP_TYPE_BUILD;
 907
 908                         link = calloc(1, sizeof(*link));
 909                         link->pkg = dummy_node;
 910                         link->next = &pkg->deponi_list;
 911                         link->prev = pkg->deponi_list.prev;
 912                         link->next->prev = link;
 913                         link->prev->next = link;
 914                         link->dep_type = DEP_TYPE_BUILD;
 915                         ++pkg->depi_count;
 916                 }
 917
 918                 if (flavors) {
 919                         do {
 920                                 flavor = strsep(&flavors, " \t");
 921                         } while (flavor && *flavor == 0);
 922                         if (flavor)
 923                                 goto again;
 924                         free(flavors);
 925                 }
 926         }
 927 }
 928
 929 /*
 930  * Query the package (at least to make sure it hasn't been truncated)
 931  * and mark it as PACKAGED if found.
 932  *
 933  * Threaded function
 934  */
 935 static void
 936 childGetBinaryDistInfo(bulk_t *bulk)
 937 {
 938         char *ptr;
 939         FILE *fp;
 940         size_t len;
 941         pkg_t *pkg;
 942         const char *cav[MAXCAC];
 943         char *repopath;
 944         char buf[1024];
 945         pid_t pid;
 946         int cac;
 947
 948         asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
 949
 950         cac = 0;
 951         cav[cac++] = PKG_BINARY;
 952         cav[cac++] = "query";
 953         cav[cac++] = "-F";
 954         cav[cac++] = repopath;
 955         cav[cac++] = "%n-%v";
 956
 957         fp = dexec_open(cav, cac, &pid, NULL, 1, 0);
 958
 959         while ((ptr = fgetln(fp, &len)) != NULL) {
 960                 if (len == 0 || ptr[len-1] != '\n')
 961                         continue;
 962                 ptr[len-1] = 0;
 963                 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
 964
 965                 pkg = pkg_find(buf);
 966                 if (pkg) {
 967                         pkg->flags |= PKGF_PACKAGED;
 968                 } else {
 969                         ddprintf(0, "Note: package scan, not in list, "
 970                                     "skipping %s\n", buf);
 971                 }
 972         }
 973         if (dexec_close(fp, pid)) {
 974                 printf("pkg query command failed for %s\n", repopath);
 975         }
 976         free(repopath);
 977 }
 978
 979 static void
 980 childOptimizeEnv(bulk_t *bulk)
 981 {
 982         char *portpath;
 983         char *ptr;
 984         FILE *fp;
 985         int line;
 986         size_t len;
 987         const char *cav[MAXCAC];
 988         pid_t pid;
 989         int cac;
 990
 991         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
 992
 993         cac = 0;
 994         cav[cac++] = MAKE_BINARY;
 995         cav[cac++] = "-C";
 996         cav[cac++] = portpath;
 997         cav[cac++] = "-V_PERL5_FROM_BIN";
 998
 999         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
1000         free(portpath);
1001
1002         line = 1;
1003         while ((ptr = fgetln(fp, &len)) != NULL) {
1004                 if (len == 0 || ptr[len-1] != '\n') {
1005                         dfatal("Bad package info for %s/%s response line %d",
1006                                bulk->s1, bulk->s2, line);
1007                 }
1008                 ptr[--len] = 0;
1009
1010                 switch(line) {
1011                 case 1:         /* _PERL5_FROM_BIN */
1012                         addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1013                         break;
1014                 default:
1015                         printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1016                         break;
1017                 }
1018                 ++line;
1019         }
1020         if (line == 1) {
1021                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1022         } else if (line != 1 + 1) {
1023                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1024         }
1025         if (dexec_close(fp, pid)) {
1026                 printf("childOptimizeEnv() failed\n");
1027         }
1028 }
1029
1030 static int
1031 scan_and_queue_dir(const char *path, const char *level1, int level)
1032 {
1033         DIR *dir;
1034         char *s1;
1035         char *s2;
1036         struct dirent *den;
1037         struct stat st;
1038         int count = 0;
1039
1040         dir = opendir(path);
1041         dassert(dir, "Cannot open dports path \"%s\"", path);
1042
1043         while ((den = readdir(dir)) != NULL) {
1044                 if (den->d_namlen == 1 && den->d_name[0] == '.')
1045                         continue;
1046                 if (den->d_namlen == 2 &&
1047                     den->d_name[0] == '.' && den->d_name[1] == '.')
1048                         continue;
1049                 asprintf(&s1, "%s/%s", path, den->d_name);
1050                 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1051                         free(s1);
1052                         continue;
1053                 }
1054                 if (level == 1) {
1055                         count += scan_and_queue_dir(s1, den->d_name, 2);
1056                         free(s1);
1057                         continue;
1058                 }
1059                 asprintf(&s2, "%s/Makefile", s1);
1060                 if (lstat(s2, &st) == 0) {
1061                         queuebulk(level1, den->d_name, NULL, NULL);
1062                         ++count;
1063                 }
1064                 free(s1);
1065                 free(s2);
1066         }
1067         closedir(dir);
1068
1069         return count;
1070 }
1071
1072 static int
1073 scan_binary_repo(const char *path)
1074 {
1075         DIR *dir;
1076         struct dirent *den;
1077         size_t len;
1078         int count;
1079
1080         count = 0;
1081         dir = opendir(path);
1082         dassert(dir, "Cannot open repository path \"%s\"", path);
1083
1084         /*
1085          * NOTE: Test includes the '.' in the suffix.
1086          */
1087         while ((den = readdir(dir)) != NULL) {
1088                 len = strlen(den->d_name);
1089                 if (len > 4 &&
1090                     strcmp(den->d_name + len - 4, UsePkgSufx) == 0) {
1091                         queuebulk(den->d_name, NULL, NULL, NULL);
1092                         ++count;
1093                 }
1094         }
1095         closedir(dir);
1096
1097         return count;
1098 }
1099
1100 #if 0
1101 static void
1102 pkgfree(pkg_t *pkg)
1103 {
1104         freestrp(&pkg->portdir);
1105         freestrp(&pkg->version);
1106         freestrp(&pkg->pkgfile);
1107         freestrp(&pkg->ignore);
1108         freestrp(&pkg->fetch_deps);
1109         freestrp(&pkg->ext_deps);
1110         freestrp(&pkg->patch_deps);
1111         freestrp(&pkg->build_deps);
1112         freestrp(&pkg->lib_deps);
1113         freestrp(&pkg->run_deps);
1114         freestrp(&pkg->pos_options);
1115         freestrp(&pkg->neg_options);
1116         freestrp(&pkg->flavors);
1117         free(pkg);
1118 }
1119 #endif