usr.bin/dsynth/pkglist.c

   1 /*
   2  * Copyright (c) 2019 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * This code uses concepts and configuration based on 'synth', by
   8  * John R. Marino <draco@marino.st>, which was written in ada.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in
  18  *    the documentation and/or other materials provided with the
  19  *    distribution.
  20  * 3. Neither the name of The DragonFly Project nor the names of its
  21  *    contributors may be used to endorse or promote products derived
  22  *    from this software without specific, prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  27  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  28  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  30  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  32  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  33  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  34  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  */
  37
  38 #include "dsynth.h"
  39
  40 #define PKG_HSIZE       32768
  41 #define PKG_HMASK       32767
  42
  43 static int parsepkglist_file(const char *path, int debugstop);
  44 static void childGetPackageInfo(bulk_t *bulk);
  45 static void childGetBinaryDistInfo(bulk_t *bulk);
  46 static void childOptimizeEnv(bulk_t *bulk);
  47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
  48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo);
  49 static void resolveDepString(pkg_t *pkg, char *depstr,
  50                         int gentopo, int dep_type);
  51 static pkg_t *processPackageListBulk(int total);
  52 static int scan_and_queue_dir(const char *path, const char *level1, int level);
  53 static int scan_binary_repo(const char *path);
  54 #if 0
  55 static void pkgfree(pkg_t *pkg);
  56 #endif
  57
  58 static int PrepareSystemFlag;
  59
  60 pkg_t *PkgHash1[PKG_HSIZE];     /* by portdir */
  61 pkg_t *PkgHash2[PKG_HSIZE];     /* by pkgfile */
  62
  63 /*
  64  * Allocate a new pkg structure plus basic initialization.
  65  */
  66 static __inline pkg_t *
  67 allocpkg(void)
  68 {
  69         pkg_t *pkg;
  70
  71         pkg = calloc(1, sizeof(*pkg));
  72         pkg->idepon_list.next = &pkg->idepon_list;
  73         pkg->idepon_list.prev = &pkg->idepon_list;
  74         pkg->deponi_list.next = &pkg->deponi_list;
  75         pkg->deponi_list.prev = &pkg->deponi_list;
  76
  77         return pkg;
  78 }
  79
  80 /*
  81  * Simple hash for lookups
  82  */
  83 static __inline int
  84 pkghash(const char *str)
  85 {
  86         int hv = 0xABC32923;
  87         while (*str) {
  88                 hv = (hv << 5) ^ *str;
  89                 ++str;
  90         }
  91         hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
  92         return (hv & PKG_HMASK);
  93 }
  94
  95 static void
  96 pkg_enter(pkg_t *pkg)
  97 {
  98         pkg_t **pkgp;
  99         pkg_t *scan;
 100
 101         if (pkg->portdir) {
 102                 pkgp = &PkgHash1[pkghash(pkg->portdir)];
 103                 while ((scan = *pkgp) != NULL) {
 104                         if (strcmp(pkg->portdir, scan->portdir) == 0)
 105                                 break;
 106                         pkgp = &scan->hnext1;
 107                 }
 108                 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD));
 109                 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
 110                         ddassert(scan->idepon_list.next == &scan->idepon_list);
 111                         ddassert(scan->deponi_list.next == &scan->deponi_list);
 112                         *pkgp = pkg;
 113                         pkg->hnext1 = scan->hnext1;
 114                         free(scan->portdir);
 115                         free(scan);
 116                         scan = NULL;
 117                 }
 118                 if (scan == NULL)
 119                         *pkgp = pkg;
 120         }
 121
 122         if (pkg->pkgfile) {
 123                 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
 124                 while ((scan = *pkgp) != NULL) {
 125                         if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
 126                                 break;
 127                         pkgp = &scan->hnext2;
 128                 }
 129                 if (scan == NULL)
 130                         *pkgp = pkg;
 131         }
 132 }
 133
 134 static pkg_t *
 135 pkg_find(const char *match)
 136 {
 137         pkg_t **pkgp;
 138         pkg_t *pkg;
 139
 140         pkgp = &PkgHash1[pkghash(match)];
 141         for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
 142                 if (strcmp(pkg->portdir, match) == 0)
 143                         return pkg;
 144         }
 145         pkgp = &PkgHash2[pkghash(match)];
 146         for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
 147                 if (strcmp(pkg->pkgfile, match) == 0)
 148                         return pkg;
 149         }
 150         return NULL;
 151 }
 152
 153 /*
 154  * Parse a specific list of ports via origin name (portdir/subdir)
 155  */
 156 pkg_t *
 157 ParsePackageList(int n, char **ary, int debugstop)
 158 {
 159         pkg_t *list;
 160         int total;
 161         int fail;
 162         int i;
 163
 164         total = 0;
 165         fail = 0;
 166         initbulk(childGetPackageInfo, MaxBulk);
 167
 168         /*
 169          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 170          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 171          */
 172         queuebulk("ports-mgmt", "pkg", NULL, "x");
 173
 174         for (i = 0; i < n; ++i) {
 175                 char *l1;
 176                 char *l2;
 177                 char *l3;
 178                 struct stat st;
 179
 180                 l1 = strdup(ary[i]);
 181                 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
 182                         total += parsepkglist_file(l1, debugstop);
 183                         continue;
 184                 }
 185
 186                 l2 = strchr(l1, '/');
 187                 if (l2 == NULL) {
 188                         printf("Bad portdir specification: %s\n", l1);
 189                         free(l1);
 190                         fail = 1;
 191                         continue;
 192                 }
 193                 *l2++ = 0;
 194                 l3 = strchr(l2, '@');
 195                 if (l3)
 196                         *l3++ = 0;
 197                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 198                 ++total;
 199                 free(l1);
 200         }
 201         printf("Processing %d ports\n", total);
 202
 203         list = processPackageListBulk(total);
 204         if (fail) {
 205                 dfatal("Bad specifications, exiting");
 206                 exit(1);
 207         }
 208
 209         return list;
 210 }
 211
 212 static
 213 int
 214 parsepkglist_file(const char *path, int debugstop)
 215 {
 216         FILE *fp;
 217         char *base;
 218         char *l1;
 219         char *l2;
 220         char *l3;
 221         size_t len;
 222         int total;
 223
 224         if ((fp = fopen(path, "r")) == NULL) {
 225                 dpanic_errno("Cannot read %s\n", path);
 226                 /* NOT REACHED */
 227                 return 0;
 228         }
 229
 230         total = 0;
 231
 232         while ((base = fgetln(fp, &len)) != NULL) {
 233                 if (len == 0 || base[len-1] != '\n')
 234                         continue;
 235                 base[--len] = 0;
 236                 l1 = strtok(base, " \t\r\n");
 237                 if (l1 == NULL) {
 238                         printf("Badly formatted pkg info line: %s\n", base);
 239                         continue;
 240                 }
 241                 l2 = strchr(l1, '/');
 242                 if (l2 == NULL) {
 243                         printf("Badly formatted specification: %s\n", l1);
 244                         continue;
 245                 }
 246                 *l2++ = 0;
 247                 l3 = strchr(l2, '@');
 248                 if (l3)
 249                         *l3++ = 0;
 250                 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
 251                 ++total;
 252         }
 253         fclose(fp);
 254
 255         return total;
 256 }
 257
 258 /*
 259  * Parse packages from the list installed on the system.
 260  */
 261 pkg_t *
 262 GetLocalPackageList(void)
 263 {
 264         pkg_t *list;
 265         FILE *fp;
 266         char *base;
 267         char *data;
 268         char *l1;
 269         char *l2;
 270         char *l3;
 271         int total;
 272         int state;
 273         size_t len;
 274
 275         PrepareSystemFlag = 1;
 276         initbulk(childGetPackageInfo, MaxBulk);
 277         total = 0;
 278         state = 0;
 279         l1 = NULL;
 280         l2 = NULL;
 281         l3 = NULL;
 282
 283         fp = popen("pkg info -a -o -A", "r");
 284
 285         /*
 286          * Always include ports-mgmt/pkg.  s4 is "x" meaning not a manual
 287          * selection, "d" meaning DEBUGSTOP mode, or NULL.
 288          */
 289         queuebulk("ports-mgmt", "pkg", NULL, "x");
 290
 291         while ((base = fgetln(fp, &len)) != NULL) {
 292                 if (len == 0 || base[len-1] != '\n')
 293                         continue;
 294                 base[--len] = 0;
 295
 296                 data = strchr(base, ':');
 297                 if (data == NULL)
 298                         continue;
 299                 *data++ = 0;
 300
 301                 base = strtok(base, " \t\r");
 302                 data = strtok(data, " \t\r");
 303
 304                 if (base == NULL || data == NULL)
 305                         continue;
 306
 307                 if (strcmp(base, "Origin") == 0) {
 308                         if (state == 1) {
 309                                 queuebulk(l1, l2, NULL, NULL);
 310                                 state = 0;
 311                                 ++total;
 312                         }
 313
 314                         if (strchr(data, '/') == NULL) {
 315                                 printf("Badly formatted origin: %s\n", l1);
 316                         }
 317                         if (l1)
 318                                 free(l1);
 319                         if (l3)
 320                                 free(l3);
 321                         l1 = strdup(data);
 322                         l2 = strchr(l1, '/');
 323                         *l2++ = 0;
 324                         l3 = strchr(l2, '@');   /* typically NULL */
 325                         if (l3) {
 326                                 *l3++ = 0;
 327                                 l3 = strdup(l3);
 328                         }
 329
 330                         /*
 331                          * Don't queue ports-mgmt/pkg twice, we already
 332                          * queued it manually.
 333                          */
 334                         if (strcmp(l1, "ports-mgmt") != 0 ||
 335                             strcmp(l2, "pkg") != 0) {
 336                                 state = 1;
 337                         }
 338                         continue;
 339                 }
 340                 if (state == 1 && strcmp(base, "flavor") == 0) {
 341                         queuebulk(l1, l2, data, NULL);
 342                         state = 0;
 343                         ++total;
 344                 }
 345         }
 346         if (state == 1) {
 347                 queuebulk(l1, l2, NULL, NULL);
 348                 /*state = 0; not needed */
 349         }
 350         if (l1)
 351                 free(l1);
 352         if (l3)
 353                 free(l3);
 354
 355         pclose(fp);
 356
 357         printf("Processing %d ports\n", total);
 358
 359         list = processPackageListBulk(total);
 360
 361         return list;
 362 }
 363
 364 pkg_t *
 365 GetFullPackageList(void)
 366 {
 367         int total;
 368
 369         initbulk(childGetPackageInfo, MaxBulk);
 370         total = scan_and_queue_dir(DPortsPath, NULL, 1);
 371         printf("Scanning %d ports\n", total);
 372
 373         return processPackageListBulk(total);
 374 }
 375
 376 /*
 377  * Caller has queued the process list for bulk operation.  We retrieve
 378  * the results and clean up the bulk operation (we may have to do a second
 379  * bulk operation so we have to be the ones to clean it up).
 380  */
 381 static pkg_t *
 382 processPackageListBulk(int total)
 383 {
 384         bulk_t *bulk;
 385         pkg_t *scan;
 386         pkg_t *list;
 387         pkg_t *dep_list;
 388         pkg_t **list_tail;
 389         int count;
 390         int stop_fail;
 391         int stop_base_list;
 392         int remove_corrupt;
 393
 394         list = NULL;
 395         list_tail = &list;
 396         count = 0;
 397         remove_corrupt = 0;
 398
 399         while ((bulk = getbulk()) != NULL) {
 400                 ++count;
 401                 if ((count & 255) == 0) {
 402                         printf("%6.2f%%\r",
 403                                 (double)count * 100.0 / (double)total + 0.001);
 404                         fflush(stdout);
 405                 }
 406                 if (bulk->list) {
 407                         *list_tail = bulk->list;
 408                         bulk->list = NULL;
 409                         while ((scan = *list_tail) != NULL) {
 410                                 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
 411                                         scan->flags |= PKGF_MANUALSEL;
 412                                 pkg_enter(scan);
 413                                 list_tail = &scan->bnext;
 414                         }
 415                 }
 416                 freebulk(bulk);
 417         }
 418         printf("100.00%%\n");
 419         printf("\nTotal %d\n", count);
 420         fflush(stdout);
 421
 422         /*
 423          * Resolve all dependencies for the related packages, potentially
 424          * adding anything that could not be found to the list.  This will
 425          * continue to issue bulk operations and process the result until
 426          * no dependencies are left.
 427          */
 428         printf("Resolving dependencies...");
 429         fflush(stdout);
 430         dep_list = list;
 431         while (dep_list) {
 432                 dep_list = resolveDeps(dep_list, &list_tail, 0);
 433         }
 434         printf("done\n");
 435
 436         donebulk();
 437
 438         /*
 439          * Generate the topology
 440          */
 441         resolveDeps(list, NULL, 1);
 442
 443         /*
 444          * Do a final count, ignore place holders.
 445          *
 446          * Also set stop_fail if appropriate.  Check for direct specifications
 447          * which fail to probe and any direct dependencies of those
 448          * specifications, but don't recurse (for now)... don't check indirect
 449          * dependencies (i.e. A -> B -> C where A is directly specified, B
 450          * is adirect dependency, and C fails to probe).
 451          */
 452         count = 0;
 453         stop_fail = 0;
 454         stop_base_list = 0;
 455         for (scan = list; scan; scan = scan->bnext) {
 456                 if ((scan->flags & PKGF_ERROR) == 0) {
 457                         ++count;
 458                 }
 459                 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) {
 460                         pkglink_t *link;
 461
 462                         /*
 463                          * Directly specified package failed to probe
 464                          */
 465                         if (scan->flags & PKGF_CORRUPT) {
 466                                 ++stop_fail;
 467                                 ++stop_base_list;
 468                         }
 469
 470                         /*
 471                          * Directly specified package had a direct dependency
 472                          * that failed to probe (don't go further).
 473                          */
 474                         PKGLIST_FOREACH(link, &scan->idepon_list) {
 475                                 if (link->pkg &&
 476                                     (link->pkg->flags & PKGF_CORRUPT)) {
 477                                         ++stop_fail;
 478                                 }
 479                         }
 480                 }
 481         }
 482         printf("Total Returned %d\n", count);
 483
 484         /*
 485          * Check to see if any PKGF_MANUALSEL packages
 486          */
 487         if (stop_fail) {
 488                 printf("%d packages failed to probe\n", stop_fail);
 489                 if (PrepareSystemFlag) {
 490                         if (stop_fail == stop_base_list) {
 491                                 printf(
 492   "prepare-system: Some of your installed packages no longer exist in\n"
 493   "dports, do you wish to continue rebuilding what does exist?\n");
 494                                 if (askyn("Continue anyway? "))
 495                                         remove_corrupt = 1;
 496                         } else {
 497                                 printf(
 498   "prepare-system: Some of your installed packages have dependencies\n"
 499   "which could not be found in dports, cannot continue, aborting\n");
 500                         }
 501                 } else {
 502                         printf("unable to continue, aborting\n");
 503                 }
 504                 if (remove_corrupt == 0)
 505                         exit(1);
 506         }
 507
 508         /*
 509          * Remove corrupt packages before continuing
 510          */
 511         if (remove_corrupt) {
 512                 list_tail = &list;
 513                 while ((scan = *list_tail) != NULL) {
 514                         if (scan->flags & PKGF_CORRUPT)
 515                                 *list_tail = scan->bnext;
 516                         else
 517                                 list_tail = &scan->bnext;
 518                 }
 519         }
 520
 521         /*
 522          * Scan our binary distributions and related dependencies looking
 523          * for any packages that have already been built.
 524          */
 525         initbulk(childGetBinaryDistInfo, MaxBulk);
 526         total = scan_binary_repo(RepositoryPath);
 527         count = 0;
 528         printf("Scanning %d packages\n", total);
 529
 530         while ((bulk = getbulk()) != NULL) {
 531                 ++count;
 532                 if ((count & 255) == 0) {
 533                         printf("%6.2f%%\r",
 534                                 (double)count * 100.0 / (double)total + 0.001);
 535                         fflush(stdout);
 536                 }
 537                 freebulk(bulk);
 538         }
 539         printf("100.00%%\n");
 540         printf("\nTotal %d\n", count);
 541         fflush(stdout);
 542         donebulk();
 543
 544         printf("all done\n");
 545
 546         return list;
 547 }
 548
 549 pkg_t *
 550 GetPkgPkg(pkg_t *list)
 551 {
 552         bulk_t *bulk;
 553         pkg_t *scan;
 554
 555         for (scan = list; scan; scan = scan->bnext) {
 556                 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
 557                         return scan;
 558         }
 559
 560         /*
 561          * This will force pkg to be built, but generally this code
 562          * is not reached because the package list processing code
 563          * adds ports-mgmt/pkg unconditionally.
 564          */
 565         initbulk(childGetPackageInfo, MaxBulk);
 566         queuebulk("ports-mgmt", "pkg", NULL, "x");
 567         bulk = getbulk();
 568         dassert(bulk, "Cannot find ports-mgmt/pkg");
 569         scan = bulk->list;
 570         bulk->list = NULL;
 571         freebulk(bulk);
 572         donebulk();
 573
 574         return scan;
 575 }
 576
 577 /*
 578  * Try to optimize the environment by supplying information that
 579  * the ports system would generally have to run stuff to get on
 580  * every package.
 581  *
 582  * See childOptimizeEnv() for the actual handling.  We execute
 583  * a single make -V... -V... for ports-mgmt/pkg from within the
 584  * bulk system (which handles the environment and disables
 585  * /etc/make.conf), and we then call addbuildenv() as appropriate.
 586  *
 587  * _PERL5_FROM_BIN
 588  * add others...
 589  */
 590 void
 591 OptimizeEnv(void)
 592 {
 593         bulk_t *bulk;
 594
 595         initbulk(childOptimizeEnv, MaxBulk);
 596         queuebulk("ports-mgmt", "pkg", NULL, NULL);
 597         bulk = getbulk();
 598         freebulk(bulk);
 599         donebulk();
 600 }
 601
 602 /*
 603  * Run through the list resolving dependencies and constructing the topology
 604  * linkages.   This may append packages to the list.  Dependencies to dummy
 605  * nodes which do not specify a flavor do not need special handling, the
 606  * search code in build.c will properly follow the first flavor.
 607  */
 608 static pkg_t *
 609 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
 610 {
 611         pkg_t *ret_list = NULL;
 612         pkg_t *scan;
 613         pkg_t *use;
 614         bulk_t *bulk;
 615
 616         for (scan = list; scan; scan = scan->bnext) {
 617                 use = pkg_find(scan->portdir);
 618                 resolveFlavors(use, scan->flavors, gentopo);
 619                 resolveDepString(use, scan->fetch_deps,
 620                                  gentopo, DEP_TYPE_FETCH);
 621                 resolveDepString(use, scan->ext_deps,
 622                                  gentopo, DEP_TYPE_EXT);
 623                 resolveDepString(use, scan->patch_deps,
 624                                  gentopo, DEP_TYPE_PATCH);
 625                 resolveDepString(use, scan->build_deps,
 626                                  gentopo, DEP_TYPE_BUILD);
 627                 resolveDepString(use, scan->lib_deps,
 628                                  gentopo, DEP_TYPE_LIB);
 629                 resolveDepString(use, scan->run_deps,
 630                                  gentopo, DEP_TYPE_RUN);
 631         }
 632
 633         /*
 634          * No bulk ops are queued when doing the final topology
 635          * generation.
 636          *
 637          * Avoid entering duplicate results from the bulk ops.  Duplicate
 638          * results are mostly filtered out, but not always.  A dummy node
 639          * representing multiple flavors will parse-out the flavors
 640          */
 641         if (gentopo)
 642                 return NULL;
 643         while ((bulk = getbulk()) != NULL) {
 644                 if (bulk->list) {
 645                         if (ret_list == NULL)
 646                                 ret_list = bulk->list;
 647                         **list_tailp = bulk->list;
 648                         bulk->list = NULL;
 649                         while ((scan = **list_tailp) != NULL) {
 650                                 pkg_enter(scan);
 651                                 *list_tailp = &scan->bnext;
 652                         }
 653                 }
 654                 freebulk(bulk);
 655         }
 656         return (ret_list);
 657 }
 658
 659 /*
 660  * Resolve a generic node that has flavors, queue to retrieve info for
 661  * each flavor and setup linkages as appropriate.
 662  */
 663 static void
 664 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo)
 665 {
 666         char *flavor_base;
 667         char *flavor_scan;
 668         char *flavor;
 669         char *portdir;
 670         char *s1;
 671         char *s2;
 672         pkg_t *dpkg;
 673         pkglink_t *link;
 674
 675         if ((pkg->flags & PKGF_DUMMY) == 0)
 676                 return;
 677         if (pkg->flavors == NULL || pkg->flavors[0] == 0)
 678                 return;
 679         flavor_base = strdup(flavors);
 680         flavor_scan = flavor_base;
 681
 682         for (;;) {
 683                 do {
 684                         flavor = strsep(&flavor_scan, " \t");
 685                 } while (flavor && *flavor == 0);
 686                 if (flavor == NULL)
 687                         break;
 688
 689                 /*
 690                  * Iterate each flavor generating "s1/s2@flavor".
 691                  *
 692                  * queuebulk() info for each flavor, and set-up the
 693                  * linkages in the topology generation pass.
 694                  */
 695                 asprintf(&portdir, "%s@%s", pkg->portdir, flavor);
 696                 s1 = strdup(pkg->portdir);
 697                 s2 = strchr(s1, '/');
 698                 *s2++ = 0;
 699
 700                 dpkg = pkg_find(portdir);
 701                 if (dpkg && gentopo) {
 702                         /*
 703                          * Setup linkages
 704                          */
 705                         free(portdir);
 706
 707                         link = calloc(1, sizeof(*link));
 708                         link->pkg = dpkg;
 709                         link->next = &pkg->idepon_list;
 710                         link->prev = pkg->idepon_list.prev;
 711                         link->next->prev = link;
 712                         link->prev->next = link;
 713                         link->dep_type = DEP_TYPE_BUILD;
 714
 715                         link = calloc(1, sizeof(*link));
 716                         link->pkg = pkg;
 717                         link->next = &dpkg->deponi_list;
 718                         link->prev = dpkg->deponi_list.prev;
 719                         link->next->prev = link;
 720                         link->prev->next = link;
 721                         link->dep_type = DEP_TYPE_BUILD;
 722                         ++dpkg->depi_count;
 723                 } else if (gentopo == 0 && dpkg == NULL) {
 724                         /*
 725                          * Use a place-holder to prevent duplicate
 726                          * dependencies from being processed.  The placeholder
 727                          * will be replaced by the actual dependency.
 728                          */
 729                         dpkg = allocpkg();
 730                         dpkg->portdir = portdir;
 731                         dpkg->flags = PKGF_PLACEHOLD;
 732                         pkg_enter(dpkg);
 733                         queuebulk(s1, s2, flavor, NULL);
 734                 }
 735                 free(s1);
 736         }
 737         free(flavor_base);
 738 }
 739
 740 static void
 741 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
 742 {
 743         char *copy_base;
 744         char *copy;
 745         char *dep;
 746         char *sep;
 747         char *tag;
 748         char *flavor;
 749         pkg_t *dpkg;
 750
 751         if (depstr == NULL || depstr[0] == 0)
 752                 return;
 753
 754         copy_base = strdup(depstr);
 755         copy = copy_base;
 756
 757         for (;;) {
 758                 do {
 759                         dep = strsep(&copy, " \t");
 760                 } while (dep && *dep == 0);
 761                 if (dep == NULL)
 762                         break;
 763
 764                 /*
 765                  * Ignore dependencies prefixed with ${NONEXISTENT}
 766                  */
 767                 if (strncmp(dep, "/nonexistent:", 13) == 0)
 768                         continue;
 769
 770                 dep = strchr(dep, ':');
 771                 if (dep == NULL || *dep != ':') {
 772                         printf("Error parsing dependency for %s: %s\n",
 773                                pkg->portdir, copy_base);
 774                         continue;
 775                 }
 776                 ++dep;
 777
 778                 /*
 779                  * Strip-off any DPortsPath prefix.  EXTRACT_DEPENDS
 780                  * often (always?) generates this prefix.
 781                  */
 782                 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
 783                         dep += strlen(DPortsPath);
 784                         if (*dep == '/')
 785                                 ++dep;
 786                 }
 787
 788                 /*
 789                  * Strip-off any tag (such as :patch).  We don't try to
 790                  * organize dependencies at this fine a grain (for now).
 791                  */
 792                 tag = strchr(dep, ':');
 793                 if (tag)
 794                         *tag++ = 0;
 795
 796                 /*
 797                  * Locate the dependency
 798                  */
 799                 if ((dpkg = pkg_find(dep)) != NULL) {
 800                         if (gentopo) {
 801                                 pkglink_t *link;
 802
 803                                 /*
 804                                  * NOTE: idep_count is calculated recursively
 805                                  *       at build-time
 806                                  */
 807                                 ddprintf(0, "Add Dependency %s -> %s\n",
 808                                         pkg->portdir, dpkg->portdir);
 809                                 link = calloc(1, sizeof(*link));
 810                                 link->pkg = dpkg;
 811                                 link->next = &pkg->idepon_list;
 812                                 link->prev = pkg->idepon_list.prev;
 813                                 link->next->prev = link;
 814                                 link->prev->next = link;
 815                                 link->dep_type = dep_type;
 816
 817                                 link = calloc(1, sizeof(*link));
 818                                 link->pkg = pkg;
 819                                 link->next = &dpkg->deponi_list;
 820                                 link->prev = dpkg->deponi_list.prev;
 821                                 link->next->prev = link;
 822                                 link->prev->next = link;
 823                                 link->dep_type = dep_type;
 824                                 ++dpkg->depi_count;
 825                         }
 826                         continue;
 827                 }
 828
 829                 /*
 830                  * This shouldn't happen because we already took a first
 831                  * pass and should have generated the pkgs.
 832                  */
 833                 if (gentopo) {
 834                         printf("Topology Generate failed for %s: %s\n",
 835                                 pkg->portdir, copy_base);
 836                         continue;
 837                 }
 838
 839                 /*
 840                  * Separate out the two dports directory components and
 841                  * extract the optional '@flavor' specification.
 842                  */
 843                 sep = strchr(dep, '/');
 844                 if (sep == NULL) {
 845                         printf("Error parsing dependency for %s: %s\n",
 846                                pkg->portdir, copy_base);
 847                         continue;
 848                 }
 849                 *sep++ = 0;
 850
 851                 /*
 852                  * The flavor hangs off the separator, not the tag
 853                  */
 854                 flavor = strrchr(sep, '@');
 855 #if 0
 856                 if (tag)
 857                         flavor = strrchr(tag, '@');
 858                 else
 859                         flavor = strrchr(sep, '@');
 860 #endif
 861                 if (flavor)
 862                         *flavor++ = 0;
 863
 864                 if (flavor)
 865                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
 866                                pkg->portdir, dep, sep, flavor);
 867                 else
 868                         ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
 869                                pkg->portdir, dep, sep);
 870
 871                 /*
 872                  * Use a place-holder to prevent duplicate dependencies from
 873                  * being processed.  The placeholder will be replaced by
 874                  * the actual dependency.
 875                  */
 876                 dpkg = allocpkg();
 877                 if (flavor)
 878                         asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
 879                 else
 880                         asprintf(&dpkg->portdir, "%s/%s", dep, sep);
 881                 dpkg->flags = PKGF_PLACEHOLD;
 882                 pkg_enter(dpkg);
 883
 884                 queuebulk(dep, sep, flavor, NULL);
 885         }
 886         free(copy_base);
 887 }
 888
 889 void
 890 FreePackageList(pkg_t *pkgs __unused)
 891 {
 892         dfatal("not implemented");
 893 }
 894
 895 /*
 896  * Scan some or all dports to allocate the related pkg structure.  Dependencies
 897  * are stored but not processed.
 898  *
 899  * Threaded function
 900  */
 901 static void
 902 childGetPackageInfo(bulk_t *bulk)
 903 {
 904         pkg_t *pkg;
 905         char *flavor;
 906         char *ptr;
 907         FILE *fp;
 908         int line;
 909         size_t len;
 910         char *portpath;
 911         char *flavarg;
 912         const char *cav[MAXCAC];
 913         pid_t pid;
 914         int cac;
 915
 916         /*
 917          * If the package has flavors we will loop on each one.  If a flavor
 918          * is not passed in s3 we will loop on all flavors, otherwise we will
 919          * only process the passed-in flavor.
 920          */
 921         flavor = bulk->s3;      /* usually NULL */
 922
 923         bulk->list = NULL;
 924
 925         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
 926         if (flavor)
 927                 asprintf(&flavarg, "FLAVOR=%s", flavor);
 928         else
 929                 flavarg = NULL;
 930
 931         cac = 0;
 932         cav[cac++] = MAKE_BINARY;
 933         cav[cac++] = "-C";
 934         cav[cac++] = portpath;
 935         if (flavarg)
 936                 cav[cac++] = flavarg;
 937         cav[cac++] = "-VPKGVERSION";
 938         cav[cac++] = "-VPKGFILE:T";
 939         cav[cac++] = "-VALLFILES";
 940         cav[cac++] = "-VDIST_SUBDIR";
 941         cav[cac++] = "-VMAKE_JOBS_NUMBER";
 942         cav[cac++] = "-VIGNORE";
 943         cav[cac++] = "-VFETCH_DEPENDS";
 944         cav[cac++] = "-VEXTRACT_DEPENDS";
 945         cav[cac++] = "-VPATCH_DEPENDS";
 946         cav[cac++] = "-VBUILD_DEPENDS";
 947         cav[cac++] = "-VLIB_DEPENDS";
 948         cav[cac++] = "-VRUN_DEPENDS";
 949         cav[cac++] = "-VSELECTED_OPTIONS";
 950         cav[cac++] = "-VDESELECTED_OPTIONS";
 951         cav[cac++] = "-VUSE_LINUX";
 952         cav[cac++] = "-VFLAVORS";
 953         cav[cac++] = "-VUSES";
 954
 955         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
 956         free(portpath);
 957         freestrp(&flavarg);
 958
 959         pkg = allocpkg();
 960         if (flavor)
 961                 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
 962         else
 963                 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
 964
 965         line = 1;
 966         while ((ptr = fgetln(fp, &len)) != NULL) {
 967                 if (len == 0 || ptr[len-1] != '\n') {
 968                         dfatal("Bad package info for %s/%s response line %d",
 969                                bulk->s1, bulk->s2, line);
 970                 }
 971                 ptr[--len] = 0;
 972
 973                 switch(line) {
 974                 case 1:         /* PKGVERSION */
 975                         asprintf(&pkg->version, "%s", ptr);
 976                         break;
 977                 case 2:         /* PKGFILE */
 978                         asprintf(&pkg->pkgfile, "%s", ptr);
 979                         break;
 980                 case 3:         /* ALLFILES (aka DISTFILES + patch files) */
 981                         asprintf(&pkg->distfiles, "%s", ptr);
 982                         break;
 983                 case 4:         /* DIST_SUBDIR */
 984                         pkg->distsubdir = strdup_or_null(ptr);
 985                         break;
 986                 case 5:         /* MAKE_JOBS_NUMBER */
 987                         pkg->make_jobs_number = strtol(ptr, NULL, 0);
 988                         break;
 989                 case 6:         /* IGNORE */
 990                         pkg->ignore = strdup_or_null(ptr);
 991                         break;
 992                 case 7:         /* FETCH_DEPENDS */
 993                         pkg->fetch_deps = strdup_or_null(ptr);
 994                         break;
 995                 case 8:         /* EXTRACT_DEPENDS */
 996                         pkg->ext_deps = strdup_or_null(ptr);
 997                         break;
 998                 case 9:         /* PATCH_DEPENDS */
 999                         pkg->patch_deps = strdup_or_null(ptr);
1000                         break;
1001                 case 10:        /* BUILD_DEPENDS */
1002                         pkg->build_deps = strdup_or_null(ptr);
1003                         break;
1004                 case 11:        /* LIB_DEPENDS */
1005                         pkg->lib_deps = strdup_or_null(ptr);
1006                         break;
1007                 case 12:        /* RUN_DEPENDS */
1008                         pkg->run_deps = strdup_or_null(ptr);
1009                         break;
1010                 case 13:        /* SELECTED_OPTIONS */
1011                         pkg->pos_options = strdup_or_null(ptr);
1012                         break;
1013                 case 14:        /* DESELECTED_OPTIONS */
1014                         pkg->neg_options = strdup_or_null(ptr);
1015                         break;
1016                 case 15:        /* USE_LINUX */
1017                         if (ptr[0])
1018                                 pkg->use_linux = 1;
1019                         break;
1020                 case 16:        /* FLAVORS */
1021                         asprintf(&pkg->flavors, "%s", ptr);
1022                         break;
1023                 case 17:        /* USES */
1024                         asprintf(&pkg->uses, "%s", ptr);
1025                         if (strstr(pkg->uses, "metaport"))
1026                                 pkg->flags |= PKGF_META;
1027                         break;
1028                 default:
1029                         printf("EXTRA LINE: %s\n", ptr);
1030                         break;
1031                 }
1032                 ++line;
1033         }
1034         if (line == 1) {
1035                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1036                 pkg->flags |= PKGF_NOTFOUND;
1037         } else if (line != 17 + 1) {
1038                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1039                 pkg->flags |= PKGF_CORRUPT;
1040         }
1041         if (dexec_close(fp, pid)) {
1042                 printf("make -V* command for %s/%s failed\n",
1043                         bulk->s1, bulk->s2);
1044                 pkg->flags |= PKGF_CORRUPT;
1045         }
1046         ddassert(bulk->s1);
1047
1048         /*
1049          * DEBUGSTOP mode
1050          */
1051         if (bulk->s4 && bulk->s4[0] == 'd')
1052                 pkg->flags |= PKGF_DEBUGSTOP;
1053
1054         /*
1055          * Mark as a dummy node, the front-end will iterate the flavors
1056          * and create sub-nodes for us.
1057          *
1058          * Get rid of elements returned that are for the first flavor.
1059          * We are creating a dummy node here, not the node for the first
1060          * flavor.
1061          */
1062         if (flavor == NULL && pkg->flavors && pkg->flavors[0]) {
1063                 pkg->flags |= PKGF_DUMMY;
1064                 freestrp(&pkg->fetch_deps);
1065                 freestrp(&pkg->ext_deps);
1066                 freestrp(&pkg->patch_deps);
1067                 freestrp(&pkg->build_deps);
1068                 freestrp(&pkg->lib_deps);
1069                 freestrp(&pkg->run_deps);
1070                 freestrp(&pkg->pkgfile);
1071         }
1072
1073         /*
1074          * Only one pkg is put on the return list now.  This code no
1075          * longer creates pseudo-nodes for flavors (the frontend requests
1076          * each flavor instead).
1077          */
1078         bulk->list = pkg;
1079 }
1080
1081 /*
1082  * Query the package (at least to make sure it hasn't been truncated)
1083  * and mark it as PACKAGED if found.
1084  *
1085  * Threaded function
1086  */
1087 static void
1088 childGetBinaryDistInfo(bulk_t *bulk)
1089 {
1090         char *ptr;
1091         FILE *fp;
1092         size_t len;
1093         pkg_t *pkg;
1094         const char *cav[MAXCAC];
1095         char *repopath;
1096         char buf[1024];
1097         pid_t pid;
1098         int cac;
1099         int deleteme;
1100
1101         asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
1102
1103         cac = 0;
1104         cav[cac++] = PKG_BINARY;
1105         cav[cac++] = "query";
1106         cav[cac++] = "-F";
1107         cav[cac++] = repopath;
1108         cav[cac++] = "%n-%v";
1109
1110         fp = dexec_open(cav, cac, &pid, NULL, 1, 0);
1111         deleteme = DeleteObsoletePkgs;
1112
1113         while ((ptr = fgetln(fp, &len)) != NULL) {
1114                 if (len == 0 || ptr[len-1] != '\n')
1115                         continue;
1116                 ptr[len-1] = 0;
1117                 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
1118
1119                 pkg = pkg_find(buf);
1120                 if (pkg) {
1121                         pkg->flags |= PKGF_PACKAGED;
1122                         deleteme = 0;
1123                 } else {
1124                         ddprintf(0, "Note: package scan, not in list, "
1125                                     "skipping %s\n", buf);
1126                 }
1127         }
1128         if (dexec_close(fp, pid)) {
1129                 printf("pkg query command failed for %s\n", repopath);
1130         }
1131         if (deleteme) {
1132                 dlog(DLOG_ALL | DLOG_STDOUT,
1133                      "Deleting obsolete package %s\n", repopath);
1134                 remove(repopath);
1135         }
1136         free(repopath);
1137 }
1138
1139 static void
1140 childOptimizeEnv(bulk_t *bulk)
1141 {
1142         char *portpath;
1143         char *ptr;
1144         FILE *fp;
1145         int line;
1146         size_t len;
1147         const char *cav[MAXCAC];
1148         pid_t pid;
1149         int cac;
1150
1151         asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
1152
1153         cac = 0;
1154         cav[cac++] = MAKE_BINARY;
1155         cav[cac++] = "-C";
1156         cav[cac++] = portpath;
1157         cav[cac++] = "-V_PERL5_FROM_BIN";
1158
1159         fp = dexec_open(cav, cac, &pid, NULL, 1, 1);
1160         free(portpath);
1161
1162         line = 1;
1163         while ((ptr = fgetln(fp, &len)) != NULL) {
1164                 if (len == 0 || ptr[len-1] != '\n') {
1165                         dfatal("Bad package info for %s/%s response line %d",
1166                                bulk->s1, bulk->s2, line);
1167                 }
1168                 ptr[--len] = 0;
1169
1170                 switch(line) {
1171                 case 1:         /* _PERL5_FROM_BIN */
1172                         addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1173                         break;
1174                 default:
1175                         printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1176                         break;
1177                 }
1178                 ++line;
1179         }
1180         if (line == 1) {
1181                 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1182         } else if (line != 1 + 1) {
1183                 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1184         }
1185         if (dexec_close(fp, pid)) {
1186                 printf("childOptimizeEnv() failed\n");
1187         }
1188 }
1189
1190 static int
1191 scan_and_queue_dir(const char *path, const char *level1, int level)
1192 {
1193         DIR *dir;
1194         char *s1;
1195         char *s2;
1196         struct dirent *den;
1197         struct stat st;
1198         int count = 0;
1199
1200         dir = opendir(path);
1201         dassert(dir, "Cannot open dports path \"%s\"", path);
1202
1203         while ((den = readdir(dir)) != NULL) {
1204                 if (den->d_namlen == 1 && den->d_name[0] == '.')
1205                         continue;
1206                 if (den->d_namlen == 2 &&
1207                     den->d_name[0] == '.' && den->d_name[1] == '.')
1208                         continue;
1209                 asprintf(&s1, "%s/%s", path, den->d_name);
1210                 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1211                         free(s1);
1212                         continue;
1213                 }
1214                 if (level == 1) {
1215                         count += scan_and_queue_dir(s1, den->d_name, 2);
1216                         free(s1);
1217                         continue;
1218                 }
1219                 asprintf(&s2, "%s/Makefile", s1);
1220                 if (lstat(s2, &st) == 0) {
1221                         queuebulk(level1, den->d_name, NULL, NULL);
1222                         ++count;
1223                 }
1224                 free(s1);
1225                 free(s2);
1226         }
1227         closedir(dir);
1228
1229         return count;
1230 }
1231
1232 static int
1233 scan_binary_repo(const char *path)
1234 {
1235         DIR *dir;
1236         struct dirent *den;
1237         size_t len;
1238         int count;
1239
1240         count = 0;
1241         dir = opendir(path);
1242         dassert(dir, "Cannot open repository path \"%s\"", path);
1243
1244         /*
1245          * NOTE: Test includes the '.' in the suffix.
1246          */
1247         while ((den = readdir(dir)) != NULL) {
1248                 len = strlen(den->d_name);
1249                 if (len > 4 &&
1250                     strcmp(den->d_name + len - 4, UsePkgSufx) == 0) {
1251                         queuebulk(den->d_name, NULL, NULL, NULL);
1252                         ++count;
1253                 }
1254         }
1255         closedir(dir);
1256
1257         return count;
1258 }
1259
1260 #if 0
1261 static void
1262 pkgfree(pkg_t *pkg)
1263 {
1264         freestrp(&pkg->portdir);
1265         freestrp(&pkg->version);
1266         freestrp(&pkg->pkgfile);
1267         freestrp(&pkg->ignore);
1268         freestrp(&pkg->fetch_deps);
1269         freestrp(&pkg->ext_deps);
1270         freestrp(&pkg->patch_deps);
1271         freestrp(&pkg->build_deps);
1272         freestrp(&pkg->lib_deps);
1273         freestrp(&pkg->run_deps);
1274         freestrp(&pkg->pos_options);
1275         freestrp(&pkg->neg_options);
1276         freestrp(&pkg->flavors);
1277         free(pkg);
1278 }
1279 #endif