usr.bin/dsynth/pkglist.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37
  38 #include "dsynth.h"
  39
  40 #define PKG_HSIZE       32768
  41 #define PKG_HMASK       32767
  42
  43 static int parsepkglist_file(const char *path, int debugstop);
  44 static void childGetPackageInfo(bulk_t *bulk);
  45 static void childGetBinaryDistInfo(bulk_t *bulk);
  46 static void childOptimizeEnv(bulk_t *bulk);
  47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
  48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo);
  49 static void resolveDepString(pkg_t *pkg, char *depstr,
  50                         int gentopo, int dep_type);
  51 static pkg_t *processPackageListBulk(int total);
  52 static int scan_and_queue_dir(const char *path, const char *level1, int level);
  53 static int scan_binary_repo(const char *path);
  54 #if 0
  55 static void pkgfree(pkg_t *pkg);
  56 #endif
  57
  58 pkg_t *PkgHash1[PKG_HSIZE];     /* by portdir */
  59 pkg_t *PkgHash2[PKG_HSIZE];     /* by pkgfile */
  60
  61 /*
  62  * Allocate a new pkg structure plus basic initialization.
  63  */
  64 static __inline pkg_t *
  65 allocpkg(void)
  66 {
  67         pkg_t *pkg;
  68
  69         pkg = calloc(1, sizeof(*pkg));
  70         pkg->idepon_list.next = &pkg->idepon_list;
  71         pkg->idepon_list.prev = &pkg->idepon_list;
  72         pkg->deponi_list.next = &pkg->deponi_list;
  73         pkg->deponi_list.prev = &pkg->deponi_list;
  74
  75         return pkg;
  76 }
  77
  78 /*
  79  * Simple hash for lookups
  80  */
  81 static __inline int
  82 pkghash(const char *str)
  83 {
  84         int hv = 0xABC32923;
  85         while (*str) {
  86                 hv = (hv << 5) ^ *str;
  87                 ++str;
  88         }
  89         hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
  90         return (hv & PKG_HMASK);
  91 }
  92
  93 static void
  94 pkg_enter(pkg_t *pkg)
  95 {
  96         pkg_t **pkgp;
  97         pkg_t *scan;
  98
  99         if (pkg->portdir) {
 100                 pkgp = &PkgHash1[pkghash(pkg->portdir)];
 101                 while ((scan = *pkgp) != NULL) {
 102                         if (strcmp(pkg->portdir, scan->portdir) == 0)
 103                                 break;
 104                         pkgp = &scan->hnext1;
 105                 }
 106                 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD));
 107                 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
 108                         ddassert(scan->idepon_list.next == &scan->idepon_list);
 109                         ddassert(scan->deponi_list.next == &scan->deponi_list);
 110                         *pkgp = pkg;
 111                         pkg->hnext1 = scan->hnext1;
 112                         free(scan->portdir);
 113                         free(scan);
 114                         scan = NULL;
 115                 }
 116                 if (scan == NULL)
 117                         *pkgp = pkg;
 118         }
 119
 120         if (pkg->pkgfile) {
 121                 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
 122                 while ((scan = *pkgp) != NULL) {
 123                         if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
 124                                 break;
 125                         pkgp = &scan->hnext2;
 126                 }
 127                 if (scan == NULL)
 128                         *pkgp = pkg;
 129         }
 130 }
 131
 132 static pkg_t *
 133 pkg_find(const char *match)
 134 {
 135         pkg_t **pkgp;
 136         pkg_t *pkg;
 137
 138         pkgp = &PkgHash1[pkghash(match)];
 139         for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
 140                 if (strcmp(pkg->portdir, match) == 0)
 141                         return pkg;
 142         }
 143         pkgp = &PkgHash2[pkghash(match)];
 144         for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
 145                 if (strcmp(pkg->pkgfile, match) == 0)
 146                         return pkg;
 147         }
 148         return NULL;
 149 }
 150
 151 /*
 152  * Parse a specific list of ports via origin name (portdir/subdir)
 153  */
 154 pkg_t *
 155 ParsePackageList(int n, char **ary, int debugstop)
 156 {
 157         pkg_t *list;
 158         int total;
 159         int fail;
 160         int i;
 161
 162         total = 0;
 163         fail = 0;
 164         initbulk(childGetPackageInfo, MaxBulk);
 165
 166         /*
 167          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 168          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 169          */
 170         queuebulk("ports-mgmt", "pkg", NULL, "x");
 171
 172         for (i = 0; i < n; ++i) {
 173                 char *l1;
 174                 char *l2;
 175                 char *l3;
 176                 struct stat st;
 177
 178                 l1 = strdup(ary[i]);
 179                 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
 180                         total += parsepkglist_file(l1, debugstop);
 181                         continue;
 182                 }
 183
 184                 l2 = strchr(l1, '/');
 185                 if (l2 == NULL) {
 186                         printf("Bad portdir specification: %s\n", l1);
 187                         free(l1);
 188                         fail = 1;
 189                         continue;
 190                 }
 191                 *l2++ = 0;
 192                 l3 = strchr(l2, '@');
 193                 if (l3)
 194                         *l3++ = 0;
 195                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 196                 ++total;
 197                 free(l1);
 198         }
 199         printf("Processing %d ports\n", total);
 200
 201         list = processPackageListBulk(total);
 202         if (fail) {
 203                 dfatal("Bad specifications, exiting");
 204                 exit(1);
 205         }
 206
 207         return list;
 208 }
 209
 210 static
 211 int
 212 parsepkglist_file(const char *path, int debugstop)
 213 {
 214         FILE *fp;
 215         char *base;
 216         char *l1;
 217         char *l2;
 218         char *l3;
 219         size_t len;
 220         int total;
 221
 222         if ((fp = fopen(path, "r")) == NULL) {
 223                 dpanic_errno("Cannot read %s\n", path);
 224                 /* NOT REACHED */
 225                 return 0;
 226         }
 227
 228         total = 0;
 229
 230         while ((base = fgetln(fp, &len)) != NULL) {
 231                 if (len == 0 || base[len-1] != '\n')
 232                         continue;
 233                 base[--len] = 0;
 234                 l1 = strtok(base, " \t\r\n");
 235                 if (l1 == NULL) {
 236                         printf("Badly formatted pkg info line: %s\n", base);
 237                         continue;
 238                 }
 239                 l2 = strchr(l1, '/');
 240                 if (l2 == NULL) {
 241                         printf("Badly formatted specification: %s\n", l1);
 242                         continue;
 243                 }
 244                 *l2++ = 0;
 245                 l3 = strchr(l2, '@');
 246                 if (l3)
 247                         *l3++ = 0;
 248                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 249                 ++total;
 250         }
 251         fclose(fp);
 252
 253         return total;
 254 }
 255
 256 /*
 257  * Parse packages from the list installed on the system.
 258  */
 259 pkg_t *
 260 GetLocalPackageList(void)
 261 {
 262         pkg_t *list;
 263         FILE *fp;
 264         char *base;
 265         char *l1;
 266         char *l2;
 267         char *l3;
 268         int total;
 269         size_t len;
 270
 271         initbulk(childGetPackageInfo, MaxBulk);
 272         total = 0;
 273
 274         fp = popen("pkg info -a -o", "r");
 275
 276         /*
 277          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 278          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 279          */
 280         queuebulk("ports-mgmt", "pkg", NULL, "x");
 281
 282         while ((base = fgetln(fp, &len)) != NULL) {
 283                 if (len == 0 || base[len-1] != '\n')
 284                         continue;
 285                 base[--len] = 0;
 286                 if (strtok(base, " \t") == NULL) {
 287                         printf("Badly formatted pkg info line: %s\n", base);
 288                         continue;
 289                 }
 290                 l1 = strtok(NULL, " \t");
 291                 if (l1 == NULL) {
 292                         printf("Badly formatted pkg info line: %s\n", base);
 293                         continue;
 294                 }
 295
 296                 l2 = strchr(l1, '/');
 297                 if (l2 == NULL) {
 298                         printf("Badly formatted specification: %s\n", l1);
 299                         continue;
 300                 }
 301                 *l2++ = 0;
 302                 l3 = strchr(l2, '@');
 303                 if (l3)
 304                         *l3++ = 0;
 305                 queuebulk(l1, l2, l3, NULL);
 306                 ++total;
 307         }
 308         pclose(fp);
 309
 310         printf("Processing %d ports\n", total);
 311
 312         list = processPackageListBulk(total);
 313
 314         return list;
 315 }
 316
 317 pkg_t *
 318 GetFullPackageList(void)
 319 {
 320         int total;
 321
 322         initbulk(childGetPackageInfo, MaxBulk);
 323         total = scan_and_queue_dir(DPortsPath, NULL, 1);
 324         printf("Scanning %d ports\n", total);
 325
 326         return processPackageListBulk(total);
 327 }
 328
 329 /*
 330  * Caller has queued the process list for bulk operation.  We retrieve
 331  * the results and clean up the bulk operation (we may have to do a second
 332  * bulk operation so we have to be the ones to clean it up).
 333  */
 334 static pkg_t *
 335 processPackageListBulk(int total)
 336 {
 337         bulk_t *bulk;
 338         pkg_t *scan;
 339         pkg_t *list;
 340         pkg_t *dep_list;
 341         pkg_t **list_tail;
 342         int count;
 343         int stop_fail;
 344
 345         list = NULL;
 346         list_tail = &list;
 347         count = 0;
 348
 349         while ((bulk = getbulk()) != NULL) {
 350                 ++count;
 351                 if ((count & 255) == 0) {
 352                         printf("%6.2f%%\r",
 353                                 (double)count * 100.0 / (double)total + 0.001);
 354                         fflush(stdout);
 355                 }
 356                 if (bulk->list) {
 357                         *list_tail = bulk->list;
 358                         bulk->list = NULL;
 359                         while ((scan = *list_tail) != NULL) {
 360                                 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
 361                                         scan->flags |= PKGF_MANUALSEL;
 362                                 pkg_enter(scan);
 363                                 list_tail = &scan->bnext;
 364                         }
 365                 }
 366                 freebulk(bulk);
 367         }
 368         printf("100.00%%\n");
 369         printf("\nTotal %d\n", count);
 370         fflush(stdout);
 371
 372         /*
 373          * Resolve all dependencies for the related packages, potentially
 374          * adding anything that could not be found to the list.  This will
 375          * continue to issue bulk operations and process the result until
 376          * no dependencies are left.
 377          */
 378         printf("Resolving dependencies...");
 379         fflush(stdout);
 380         dep_list = list;
 381         while (dep_list) {
 382                 dep_list = resolveDeps(dep_list, &list_tail, 0);
 383         }
 384         printf("done\n");
 385
 386         donebulk();
 387
 388         /*
 389          * Generate the topology
 390          */
 391         resolveDeps(list, NULL, 1);
 392
 393         /*
 394          * Do a final count, ignore place holders.
 395          *
 396          * Also set stop_fail if appropriate.  Check for direct specifications
 397          * which fail to probe and any direct dependencies of those
 398          * specifications, but don't recurse (for now)... don't check indirect
 399          * dependencies (i.e. A -> B -> C where A is directly specified, B
 400          * is adirect dependency, and C fails to probe).
 401          */
 402         count = 0;
 403         stop_fail = 0;
 404         for (scan = list; scan; scan = scan->bnext) {
 405                 if ((scan->flags & PKGF_ERROR) == 0) {
 406                         ++count;
 407                 }
 408                 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) {
 409                         pkglink_t *link;
 410
 411                         /*
 412                          * Directly specified package failed to probe
 413                          */
 414                         if (scan->flags & PKGF_CORRUPT)
 415                                 ++stop_fail;
 416
 417                         /*
 418                          * Directly specified package had a direct dependency
 419                          * that failed to probe (don't go further).
 420                          */
 421                         PKGLIST_FOREACH(link, &scan->idepon_list) {
 422                                 if (link->pkg &&
 423                                     (link->pkg->flags & PKGF_CORRUPT)) {
 424                                         ++stop_fail;
 425                                 }
 426                         }
 427                 }
 428         }
 429         printf("Total Returned %d\n", count);
 430
 431         /*
 432          * Check to see if any PKGF_MANUALSEL packages
 433          */
 434         if (stop_fail) {
 435                 printf("Aborting, %d packages failed to probe\n", stop_fail);
 436                 exit(1);
 437         }
 438
 439         /*
 440          * Scan our binary distributions and related dependencies looking
 441          * for any packages that have already been built.
 442          */
 443         initbulk(childGetBinaryDistInfo, MaxBulk);
 444         total = scan_binary_repo(RepositoryPath);
 445         count = 0;
 446         printf("Scanning %d packages\n", total);
 447
 448         while ((bulk = getbulk()) != NULL) {
 449                 ++count;
 450                 if ((count & 255) == 0) {
 451                         printf("%6.2f%%\r",
 452                                 (double)count * 100.0 / (double)total + 0.001);
 453                         fflush(stdout);
 454                 }
 455                 freebulk(bulk);
 456         }
 457         printf("100.00%%\n");
 458         printf("\nTotal %d\n", count);
 459         fflush(stdout);
 460         donebulk();
 461
 462         printf("all done\n");
 463
 464         return list;
 465 }
 466
 467 pkg_t *
 468 GetPkgPkg(pkg_t *list)
 469 {
 470         bulk_t *bulk;
 471         pkg_t *scan;
 472
 473         for (scan = list; scan; scan = scan->bnext) {
 474                 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
 475                         return scan;
 476         }
 477
 478         /*
 479          * This will force pkg to be built, but generally this code
 480          * is not reached because the package list processing code
 481          * adds ports-mgmt/pkg unconditionally.
 482          */
 483         initbulk(childGetPackageInfo, MaxBulk);
 484         queuebulk("ports-mgmt", "pkg", NULL, "x");
 485         bulk = getbulk();
 486         dassert(bulk, "Cannot find ports-mgmt/pkg");
 487         scan = bulk->list;
 488         bulk->list = NULL;
 489         freebulk(bulk);
 490         donebulk();
 491
 492         return scan;
 493 }
 494
 495 /*
 496  * Try to optimize the environment by supplying information that
 497  * the ports system would generally have to run stuff to get on
 498  * every package.
 499  *
 500  * See childOptimizeEnv() for the actual handling.  We execute
 501  * a single make -V... -V... for ports-mgmt/pkg from within the
 502  * bulk system (which handles the environment and disables
 503  * /etc/make.conf), and we then call addbuildenv() as appropriate.
 504  *
 505  * _PERL5_FROM_BIN
 506  * add others...
 507  */
 508 void
 509 OptimizeEnv(void)
 510 {
 511         bulk_t *bulk;
 512
 513         initbulk(childOptimizeEnv, MaxBulk);
 514         queuebulk("ports-mgmt", "pkg", NULL, NULL);
 515         bulk = getbulk();
 516         freebulk(bulk);
 517         donebulk();
 518 }
 519
 520 /*
 521  * Run through the list resolving dependencies and constructing the topology
 522  * linkages.   This may append packages to the list.  Dependencies to dummy
 523  * nodes which do not specify a flavor do not need special handling, the
 524  * search code in build.c will properly follow the first flavor.
 525  */
 526 static pkg_t *
 527 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
 528 {
 529         pkg_t *ret_list = NULL;
 530         pkg_t *scan;
 531         pkg_t *use;
 532         bulk_t *bulk;
 533
 534         for (scan = list; scan; scan = scan->bnext) {
 535                 use = pkg_find(scan->portdir);
 536                 resolveFlavors(use, scan->flavors, gentopo);
 537                 resolveDepString(use, scan->fetch_deps,
 538                                  gentopo, DEP_TYPE_FETCH);
 539                 resolveDepString(use, scan->ext_deps,
 540                                  gentopo, DEP_TYPE_EXT);
 541                 resolveDepString(use, scan->patch_deps,
 542                                  gentopo, DEP_TYPE_PATCH);
 543                 resolveDepString(use, scan->build_deps,
 544                                  gentopo, DEP_TYPE_BUILD);
 545                 resolveDepString(use, scan->lib_deps,
 546                                  gentopo, DEP_TYPE_LIB);
 547                 resolveDepString(use, scan->run_deps,
 548                                  gentopo, DEP_TYPE_RUN);
 549         }
 550
 551         /*
 552          * No bulk ops are queued when doing the final topology
 553          * generation.
 554          *
 555          * Avoid entering duplicate results from the bulk ops.  Duplicate
 556          * results are mostly filtered out, but not always.  A dummy node
 557          * representing multiple flavors will parse-out the flavors
 558          */
 559         if (gentopo)
 560                 return NULL;
 561         while ((bulk = getbulk()) != NULL) {
 562                 if (bulk->list) {
 563                         if (ret_list == NULL)
 564                                 ret_list = bulk->list;
 565                         **list_tailp = bulk->list;
 566                         bulk->list = NULL;
 567                         while ((scan = **list_tailp) != NULL) {
 568                                 pkg_enter(scan);
 569                                 *list_tailp = &scan->bnext;
 570                         }
 571                 }
 572                 freebulk(bulk);
 573         }
 574         return (ret_list);
 575 }
 576
 577 /*
 578  * Resolve a generic node that has flavors, queue to retrieve info for
 579  * each flavor and setup linkages as appropriate.
 580  */
 581 static void
 582 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo)
 583 {
 584         char *flavor_base;
 585         char *flavor_scan;
 586         char *flavor;
 587         char *portdir;
 588         char *s1;
 589         char *s2;
 590         pkg_t *dpkg;
 591         pkglink_t *link;
 592
 593         if ((pkg->flags & PKGF_DUMMY) == 0)
 594                 return;
 595         if (pkg->flavors == NULL || pkg->flavors[0] == 0)
 596                 return;
 597         flavor_base = strdup(flavors);
 598         flavor_scan = flavor_base;
 599
 600         for (;;) {
 601                 do {
 602                         flavor = strsep(&flavor_scan, " \t");
 603                 } while (flavor && *flavor == 0);
 604                 if (flavor == NULL)
 605                         break;
 606
 607                 /*
 608                  * Iterate each flavor generating "s1/s2@flavor".
 609                  *
 610                  * queuebulk() info for each flavor, and set-up the
 611                  * linkages in the topology generation pass.
 612                  */
 613                 asprintf(&portdir, "%s@%s", pkg->portdir, flavor);
 614                 s1 = strdup(pkg->portdir);
 615                 s2 = strchr(s1, '/');
 616                 *s2++ = 0;
 617
 618                 dpkg = pkg_find(portdir);
 619                 if (dpkg && gentopo) {
 620                         /*
 621                          * Setup linkages
 622                          */
 623                         free(portdir);
 624
 625                         link = calloc(1, sizeof(*link));
 626                         link->pkg = dpkg;
 627                         link->next = &pkg->idepon_list;
 628                         link->prev = pkg->idepon_list.prev;
 629                         link->next->prev = link;
 630                         link->prev->next = link;
 631                         link->dep_type = DEP_TYPE_BUILD;
 632
 633                         link = calloc(1, sizeof(*link));
 634                         link->pkg = pkg;
 635                         link->next = &dpkg->deponi_list;
 636                         link->prev = dpkg->deponi_list.prev;
 637                         link->next->prev = link;
 638                         link->prev->next = link;
 639                         link->dep_type = DEP_TYPE_BUILD;
 640                         ++dpkg->depi_count;
 641                 } else if (gentopo == 0 && dpkg == NULL) {
 642                         /*
 643                          * Use a place-holder to prevent duplicate
 644                          * dependencies from being processed.  The placeholder
 645                          * will be replaced by the actual dependency.
 646                          */
 647                         dpkg = allocpkg();
 648                         dpkg->portdir = portdir;
 649                         dpkg->flags = PKGF_PLACEHOLD;
 650                         pkg_enter(dpkg);
 651                         queuebulk(s1, s2, flavor, NULL);
 652                 }
 653                 free(s1);
 654         }
 655         free(flavor_base);
 656 }
 657
 658 static void
 659 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
 660 {
 661         char *copy_base;
 662         char *copy;
 663         char *dep;
 664         char *sep;
 665         char *tag;
 666         char *flavor;
 667         pkg_t *dpkg;
 668
 669         if (depstr == NULL || depstr[0] == 0)
 670                 return;
 671
 672         copy_base = strdup(depstr);
 673         copy = copy_base;
 674
 675         for (;;) {
 676                 do {
 677                         dep = strsep(&copy, " \t");
 678                 } while (dep && *dep == 0);
 679                 if (dep == NULL)
 680                         break;
 681
 682                 /*
 683                  * Ignore dependencies prefixed with ${NONEXISTENT}
 684                  */
 685                 if (strncmp(dep, "/nonexistent:", 13) == 0)
 686                         continue;
 687
 688                 dep = strchr(dep, ':');
 689                 if (dep == NULL || *dep != ':') {
 690                         printf("Error parsing dependency for %s: %s\n",
 691                                pkg->portdir, copy_base);
 692                         continue;
 693                 }
 694                 ++dep;
 695
 696                 /*
 697                  * Strip-off any DPortsPath prefix.  EXTRACT_DEPENDS
 698                  * often (always?) generates this prefix.
 699                  */
 700                 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
 701                         dep += strlen(DPortsPath);
 702                         if (*dep == '/')
 703                                 ++dep;
 704                 }
 705
 706                 /*
 707                  * Strip-off any tag (such as :patch).  We don't try to
 708                  * organize dependencies at this fine a grain (for now).
 709                  */
 710                 tag = strchr(dep, ':');
 711                 if (tag)
 712                         *tag++ = 0;
 713
 714                 /*
 715                  * Locate the dependency
 716                  */
 717                 if ((dpkg = pkg_find(dep)) != NULL) {
 718                         if (gentopo) {
 719                                 pkglink_t *link;
 720
 721                                 /*
 722                                  * NOTE: idep_count is calculated recursively
 723                                  *       at build-time
 724                                  */
 725                                 ddprintf(0, "Add Dependency %s -> %s\n",
 726                                         pkg->portdir, dpkg->portdir);
 727                                 link = calloc(1, sizeof(*link));
 728                                 link->pkg = dpkg;
 729                                 link->next = &pkg->idepon_list;
 730                                 link->prev = pkg->idepon_list.prev;
 731                                 link->next->prev = link;
 732                                 link->prev->next = link;
 733                                 link->dep_type = dep_type;
 734
 735                                 link = calloc(1, sizeof(*link));
 736                                 link->pkg = pkg;
 737                                 link->next = &dpkg->deponi_list;
 738                                 link->prev = dpkg->deponi_list.prev;
 739                                 link->next->prev = link;
 740                                 link->prev->next = link;
 741                                 link->dep_type = dep_type;
 742                                 ++dpkg->depi_count;
 743                         }
 744                         continue;
 745                 }
 746
 747                 /*
 748                  * This shouldn't happen because we already took a first
 749                  * pass and should have generated the pkgs.
 750                  */
 751                 if (gentopo) {
 752                         printf("Topology Generate failed for %s: %s\n",
 753                                 pkg->portdir, copy_base);
 754                         continue;
 755                 }
 756
 757                 /*
 758                  * Separate out the two dports directory components and
 759                  * extract the optional '@flavor' specification.
 760                  */
 761                 sep = strchr(dep, '/');
 762                 if (sep == NULL) {
 763                         printf("Error parsing dependency for %s: %s\n",
 764                                pkg->portdir, copy_base);
 765                         continue;
 766                 }
 767                 *sep++ = 0;
 768
 769                 /*
 770                  * The flavor hangs off the separator, not the tag
 771                  */
 772                 flavor = strrchr(sep, '@');
 773 #if 0
 774                 if (tag)
 775                         flavor = strrchr(tag, '@');
 776                 else
 777                         flavor = strrchr(sep, '@');
 778 #endif
 779                 if (flavor)
 780                         *flavor++ = 0;
 781
 782                 if (flavor)
 783                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
 784                                pkg->portdir, dep, sep, flavor);
 785                 else
 786                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
 787                                pkg->portdir, dep, sep);
 788
 789                 /*
 790                  * Use a place-holder to prevent duplicate dependencies from
 791                  * being processed.  The placeholder will be replaced by
 792                  * the actual dependency.
 793                  */
 794                 dpkg = allocpkg();
 795                 if (flavor)
 796                         asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
 797                 else
 798                         asprintf(&dpkg->portdir, "%s/%s", dep, sep);
 799                 dpkg->flags = PKGF_PLACEHOLD;
 800                 pkg_enter(dpkg);
 801
 802                 queuebulk(dep, sep, flavor, NULL);
 803         }
 804         free(copy_base);
 805 }
 806
 807 void
 808 FreePackageList(pkg_t *pkgs __unused)
 809 {
 810         dfatal("not implemented");
 811 }
 812
 813 /*
 814  * Scan some or all dports to allocate the related pkg structure.  Dependencies
 815  * are stored but not processed.
 816  *
 817  * Threaded function
 818  */
 819 static void
 820 childGetPackageInfo(bulk_t *bulk)
 821 {
 822         pkg_t *pkg;
 823         char *flavor;
 824         char *ptr;
 825         FILE *fp;
 826         int line;
 827         size_t len;
 828         char *portpath;
 829         char *flavarg;
 830         const char *cav[MAXCAC];
 831         pid_t pid;
 832         int cac;
 833
 834         /*
 835          * If the package has flavors we will loop on each one.  If a flavor
 836          * is not passed in s3 we will loop on all flavors, otherwise we will
 837          * only process the passed-in flavor.
 838          */
 839         flavor = bulk->s3;      /* usually NULL */
 840
 841         bulk->list = NULL;
 842
 843         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
 844         if (flavor)
 845                 asprintf(&flavarg, "FLAVOR=%s", flavor);
 846         else
 847                 flavarg = NULL;
 848
 849         cac = 0;
 850         cav[cac++] = MAKE_BINARY;
 851         cav[cac++] = "-C";
 852         cav[cac++] = portpath;
 853         if (flavarg)
 854                 cav[cac++] = flavarg;
 855         cav[cac++] = "-VPKGVERSION";
 856         cav[cac++] = "-VPKGFILE:T";
 857         cav[cac++] = "-VALLFILES";
 858         cav[cac++] = "-VDIST_SUBDIR";
 859         cav[cac++] = "-VMAKE_JOBS_NUMBER";
 860         cav[cac++] = "-VIGNORE";
 861         cav[cac++] = "-VFETCH_DEPENDS";
 862         cav[cac++] = "-VEXTRACT_DEPENDS";
 863         cav[cac++] = "-VPATCH_DEPENDS";
 864         cav[cac++] = "-VBUILD_DEPENDS";
 865         cav[cac++] = "-VLIB_DEPENDS";
 866         cav[cac++] = "-VRUN_DEPENDS";
 867         cav[cac++] = "-VSELECTED_OPTIONS";
 868         cav[cac++] = "-VDESELECTED_OPTIONS";
 869         cav[cac++] = "-VUSE_LINUX";
 870         cav[cac++] = "-VFLAVORS";
 871         cav[cac++] = "-VUSES";
 872
 873         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
 874         free(portpath);
 875         freestrp(&flavarg);
 876
 877         pkg = allocpkg();
 878         if (flavor)
 879                 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
 880         else
 881                 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
 882
 883         line = 1;
 884         while ((ptr = fgetln(fp, &len)) != NULL) {
 885                 if (len == 0 || ptr[len-1] != '\n') {
 886                         dfatal("Bad package info for %s/%s response line %d",
 887                                bulk->s1, bulk->s2, line);
 888                 }
 889                 ptr[--len] = 0;
 890
 891                 switch(line) {
 892                 case 1:         /* PKGVERSION */
 893                         asprintf(&pkg->version, "%s", ptr);
 894                         break;
 895                 case 2:         /* PKGFILE */
 896                         asprintf(&pkg->pkgfile, "%s", ptr);
 897                         break;
 898                 case 3:         /* ALLFILES (aka DISTFILES + patch files) */
 899                         asprintf(&pkg->distfiles, "%s", ptr);
 900                         break;
 901                 case 4:         /* DIST_SUBDIR */
 902                         pkg->distsubdir = strdup_or_null(ptr);
 903                         break;
 904                 case 5:         /* MAKE_JOBS_NUMBER */
 905                         pkg->make_jobs_number = strtol(ptr, NULL, 0);
 906                         break;
 907                 case 6:         /* IGNORE */
 908                         pkg->ignore = strdup_or_null(ptr);
 909                         break;
 910                 case 7:         /* FETCH_DEPENDS */
 911                         pkg->fetch_deps = strdup_or_null(ptr);
 912                         break;
 913                 case 8:         /* EXTRACT_DEPENDS */
 914                         pkg->ext_deps = strdup_or_null(ptr);
 915                         break;
 916                 case 9:         /* PATCH_DEPENDS */
 917                         pkg->patch_deps = strdup_or_null(ptr);
 918                         break;
 919                 case 10:        /* BUILD_DEPENDS */
 920                         pkg->build_deps = strdup_or_null(ptr);
 921                         break;
 922                 case 11:        /* LIB_DEPENDS */
 923                         pkg->lib_deps = strdup_or_null(ptr);
 924                         break;
 925                 case 12:        /* RUN_DEPENDS */
 926                         pkg->run_deps = strdup_or_null(ptr);
 927                         break;
 928                 case 13:        /* SELECTED_OPTIONS */
 929                         pkg->pos_options = strdup_or_null(ptr);
 930                         break;
 931                 case 14:        /* DESELECTED_OPTIONS */
 932                         pkg->neg_options = strdup_or_null(ptr);
 933                         break;
 934                 case 15:        /* USE_LINUX */
 935                         if (ptr[0])
 936                                 pkg->use_linux = 1;
 937                         break;
 938                 case 16:        /* FLAVORS */
 939                         asprintf(&pkg->flavors, "%s", ptr);
 940                         break;
 941                 case 17:        /* USES */
 942                         asprintf(&pkg->uses, "%s", ptr);
 943                         if (strstr(pkg->uses, "metaport"))
 944                                 pkg->flags |= PKGF_META;
 945                         break;
 946                 default:
 947                         printf("EXTRA LINE: %s\n", ptr);
 948                         break;
 949                 }
 950                 ++line;
 951         }
 952         if (line == 1) {
 953                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
 954                 pkg->flags |= PKGF_NOTFOUND;
 955         } else if (line != 17 + 1) {
 956                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
 957                 pkg->flags |= PKGF_CORRUPT;
 958         }
 959         if (dexec_close(fp, pid)) {
 960                 printf("make -V* command for %s/%s failed\n",
 961                         bulk->s1, bulk->s2);
 962                 pkg->flags |= PKGF_CORRUPT;
 963         }
 964         ddassert(bulk->s1);
 965
 966         /*
 967          * DEBUGSTOP mode
 968          */
 969         if (bulk->s4 && bulk->s4[0] == 'd')
 970                 pkg->flags |= PKGF_DEBUGSTOP;
 971
 972         /*
 973          * Mark as a dummy node, the front-end will iterate the flavors
 974          * and create sub-nodes for us.
 975          *
 976          * Get rid of elements returned that are for the first flavor.
 977          * We are creating a dummy node here, not the node for the first
 978          * flavor.
 979          */
 980         if (flavor == NULL && pkg->flavors && pkg->flavors[0]) {
 981                 pkg->flags |= PKGF_DUMMY;
 982                 freestrp(&pkg->fetch_deps);
 983                 freestrp(&pkg->ext_deps);
 984                 freestrp(&pkg->patch_deps);
 985                 freestrp(&pkg->build_deps);
 986                 freestrp(&pkg->lib_deps);
 987                 freestrp(&pkg->run_deps);
 988                 freestrp(&pkg->pkgfile);
 989         }
 990
 991         /*
 992          * Only one pkg is put on the return list now.  This code no
 993          * longer creates pseudo-nodes for flavors (the frontend requests
 994          * each flavor instead).
 995          */
 996         bulk->list = pkg;
 997 }
 998
 999 /*
1000  * Query the package (at least to make sure it hasn't been truncated)
1001  * and mark it as PACKAGED if found.
1002  *
1003  * Threaded function
1004  */
1005 static void
1006 childGetBinaryDistInfo(bulk_t *bulk)
1007 {
1008         char *ptr;
1009         FILE *fp;
1010         size_t len;
1011         pkg_t *pkg;
1012         const char *cav[MAXCAC];
1013         char *repopath;
1014         char buf[1024];
1015         pid_t pid;
1016         int cac;
1017         int deleteme;
1018
1019         asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
1020
1021         cac = 0;
1022         cav[cac++] = PKG_BINARY;
1023         cav[cac++] = "query";
1024         cav[cac++] = "-F";
1025         cav[cac++] = repopath;
1026         cav[cac++] = "%n-%v";
1027
1028         fp = dexec_open(cav, cac, &pid, NULL, 1, 0);
1029         deleteme = DeleteObsoletePkgs;
1030
1031         while ((ptr = fgetln(fp, &len)) != NULL) {
1032                 if (len == 0 || ptr[len-1] != '\n')
1033                         continue;
1034                 ptr[len-1] = 0;
1035                 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
1036
1037                 pkg = pkg_find(buf);
1038                 if (pkg) {
1039                         pkg->flags |= PKGF_PACKAGED;
1040                         deleteme = 0;
1041                 } else {
1042                         ddprintf(0, "Note: package scan, not in list, "
1043                                     "skipping %s\n", buf);
1044                 }
1045         }
1046         if (dexec_close(fp, pid)) {
1047                 printf("pkg query command failed for %s\n", repopath);
1048         }
1049         if (deleteme) {
1050                 dlog(DLOG_ALL | DLOG_STDOUT,
1051                      "Deleting obsolete package %s\n", repopath);
1052                 remove(repopath);
1053         }
1054         free(repopath);
1055 }
1056
1057 static void
1058 childOptimizeEnv(bulk_t *bulk)
1059 {
1060         char *portpath;
1061         char *ptr;
1062         FILE *fp;
1063         int line;
1064         size_t len;
1065         const char *cav[MAXCAC];
1066         pid_t pid;
1067         int cac;
1068
1069         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
1070
1071         cac = 0;
1072         cav[cac++] = MAKE_BINARY;
1073         cav[cac++] = "-C";
1074         cav[cac++] = portpath;
1075         cav[cac++] = "-V_PERL5_FROM_BIN";
1076
1077         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
1078         free(portpath);
1079
1080         line = 1;
1081         while ((ptr = fgetln(fp, &len)) != NULL) {
1082                 if (len == 0 || ptr[len-1] != '\n') {
1083                         dfatal("Bad package info for %s/%s response line %d",
1084                                bulk->s1, bulk->s2, line);
1085                 }
1086                 ptr[--len] = 0;
1087
1088                 switch(line) {
1089                 case 1:         /* _PERL5_FROM_BIN */
1090                         addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1091                         break;
1092                 default:
1093                         printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1094                         break;
1095                 }
1096                 ++line;
1097         }
1098         if (line == 1) {
1099                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1100         } else if (line != 1 + 1) {
1101                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1102         }
1103         if (dexec_close(fp, pid)) {
1104                 printf("childOptimizeEnv() failed\n");
1105         }
1106 }
1107
1108 static int
1109 scan_and_queue_dir(const char *path, const char *level1, int level)
1110 {
1111         DIR *dir;
1112         char *s1;
1113         char *s2;
1114         struct dirent *den;
1115         struct stat st;
1116         int count = 0;
1117
1118         dir = opendir(path);
1119         dassert(dir, "Cannot open dports path \"%s\"", path);
1120
1121         while ((den = readdir(dir)) != NULL) {
1122                 if (den->d_namlen == 1 && den->d_name[0] == '.')
1123                         continue;
1124                 if (den->d_namlen == 2 &&
1125                     den->d_name[0] == '.' && den->d_name[1] == '.')
1126                         continue;
1127                 asprintf(&s1, "%s/%s", path, den->d_name);
1128                 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1129                         free(s1);
1130                         continue;
1131                 }
1132                 if (level == 1) {
1133                         count += scan_and_queue_dir(s1, den->d_name, 2);
1134                         free(s1);
1135                         continue;
1136                 }
1137                 asprintf(&s2, "%s/Makefile", s1);
1138                 if (lstat(s2, &st) == 0) {
1139                         queuebulk(level1, den->d_name, NULL, NULL);
1140                         ++count;
1141                 }
1142                 free(s1);
1143                 free(s2);
1144         }
1145         closedir(dir);
1146
1147         return count;
1148 }
1149
1150 static int
1151 scan_binary_repo(const char *path)
1152 {
1153         DIR *dir;
1154         struct dirent *den;
1155         size_t len;
1156         int count;
1157
1158         count = 0;
1159         dir = opendir(path);
1160         dassert(dir, "Cannot open repository path \"%s\"", path);
1161
1162         /*
1163          * NOTE: Test includes the '.' in the suffix.
1164          */
1165         while ((den = readdir(dir)) != NULL) {
1166                 len = strlen(den->d_name);
1167                 if (len > 4 &&
1168                     strcmp(den->d_name + len - 4, UsePkgSufx) == 0) {
1169                         queuebulk(den->d_name, NULL, NULL, NULL);
1170                         ++count;
1171                 }
1172         }
1173         closedir(dir);
1174
1175         return count;
1176 }
1177
1178 #if 0
1179 static void
1180 pkgfree(pkg_t *pkg)
1181 {
1182         freestrp(&pkg->portdir);
1183         freestrp(&pkg->version);
1184         freestrp(&pkg->pkgfile);
1185         freestrp(&pkg->ignore);
1186         freestrp(&pkg->fetch_deps);
1187         freestrp(&pkg->ext_deps);
1188         freestrp(&pkg->patch_deps);
1189         freestrp(&pkg->build_deps);
1190         freestrp(&pkg->lib_deps);
1191         freestrp(&pkg->run_deps);
1192         freestrp(&pkg->pos_options);
1193         freestrp(&pkg->neg_options);
1194         freestrp(&pkg->flavors);
1195         free(pkg);
1196 }
1197 #endif