usr.bin/dfregress/runlist.c

   1 /*
   2  * Copyright (c) 2011 Alex Hornung <alex@alexhornung.com>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  19  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  20  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  22  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  23  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  25  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  26  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/resource.h>
  31 #include <sys/time.h>
  32 #include <sys/types.h>
  33 #include <sys/wait.h>
  34
  35 #include <errno.h>
  36 #include <limits.h>
  37 #include <fcntl.h>
  38 #include <signal.h>
  39 #include <stdio.h>
  40 #include <stdlib.h>
  41 #include <stdint.h>
  42 #include <string.h>
  43 #include <unistd.h>
  44 #include <pwd.h>
  45
  46 #include <err.h>
  47
  48 #include <libprop/proplib.h>
  49
  50 #include "parser.h"
  51 #include "testcase.h"
  52 #include "runlist.h"
  53 #include "userland.h"
  54 #include "kernel.h"
  55 #include <dfregress.h>
  56
  57 char output_file[PATH_MAX+1];
  58 char testcase_dir[PATH_MAX+1];
  59 char prepost_dir[PATH_MAX+1];
  60
  61 prop_array_t
  62 runlist_load_from_text(const char *runlist_file)
  63 {
  64         prop_array_t runlist;
  65
  66         runlist = prop_array_create_with_capacity(2048);
  67
  68         process_file(runlist_file, testcase_entry_parser, runlist, NULL);
  69
  70         return runlist;
  71 }
  72
  73 prop_array_t
  74 runlist_load(const char *runlist_file)
  75 {
  76         return prop_array_internalize_from_file(runlist_file);
  77 }
  78
  79 int
  80 runlist_save(const char *runlist_file, prop_array_t runlist)
  81 {
  82         return !prop_array_externalize_to_file(runlist, runlist_file);
  83 }
  84
  85 int
  86 runlist_iterate(prop_array_t runlist, runlist_iterator_t iterator, void *arg)
  87 {
  88         prop_object_iterator_t it;
  89         prop_dictionary_t testcase;
  90         int r = 0;
  91
  92         it = prop_array_iterator(runlist);
  93         if (it == NULL)
  94                 err(1, "could not get runlist iterator");
  95
  96         while ((testcase = prop_object_iterator_next(it)) != NULL) {
  97                 r = iterator(arg, testcase);
  98                 if (r != 0)
  99                         break;
 100         }
 101
 102         prop_object_iterator_release(it);
 103         return r;
 104 }
 105
 106 int
 107 runlist_run_test(void *arg, prop_dictionary_t testcase)
 108 {
 109         prop_array_t runlist = (prop_array_t)arg;
 110         struct testcase_result tr;
 111         char testcase_path[FILENAME_MAX];
 112         char testcase_dir_only[FILENAME_MAX];
 113         char prepost_path[FILENAME_MAX];
 114         char errbuf[FILENAME_MAX*2];
 115         int r, nopre, nopost;
 116         char *str;
 117
 118         sprintf(testcase_path, "%s/%s", testcase_dir,
 119             testcase_get_name(testcase));
 120         strcpy(testcase_dir_only, testcase_path);
 121         str = strrchr(testcase_dir_only, '/');
 122         if (str != NULL)
 123                 *str = '\0';
 124
 125         printf("Running testcase %s... ", testcase_get_name(testcase));
 126         fflush(stdout);
 127
 128         /* Switch to testcase directory */
 129         r = chdir(testcase_dir_only);
 130         if (r < 0) {
 131                 sprintf(errbuf, "could not switch working directory to %s: %s\n",
 132                     testcase_dir_only, strerror(errno));
 133                 testcase_set_result(testcase, RESULT_PREFAIL);
 134                 testcase_set_sys_buf(testcase, errbuf);
 135                 goto out;
 136         }
 137
 138         /* build unless nobuild flag is set */
 139         if ((testcase_get_flags(testcase) & TESTCASE_NOBUILD) == 0) {
 140                 r = run_simple_cmd(testcase_get_make_cmd(testcase), NULL,
 141                     errbuf, sizeof(errbuf), &tr);
 142                 if (r != 0) {
 143                         testcase_set_sys_buf(testcase, errbuf);
 144                         testcase_set_result(testcase, RESULT_PREFAIL);
 145                         goto out;
 146                 }
 147
 148                 if (tr.stdout_buf != NULL) {
 149                         testcase_set_build_buf(testcase, tr.stdout_buf);
 150                         free(tr.stdout_buf);
 151                 }
 152
 153                 if (tr.result != RESULT_PASS) {
 154                         if (testcase_get_type(testcase)
 155                             == TESTCASE_TYPE_BUILDONLY)
 156                                 testcase_set_result(testcase, tr.result);
 157                         else
 158                                 testcase_set_result(testcase, RESULT_BUILDFAIL);
 159
 160                         testcase_set_exit_value(testcase, tr.exit_value);
 161                         testcase_set_signal(testcase, tr.signal);
 162
 163                         goto out;
 164                 }
 165         }
 166
 167
 168         /* Pre-execution run */
 169         switch (testcase_get_precmd_type(testcase)) {
 170         case TESTCASE_INT_PRE:
 171                 /* Test case has internal but explicit PRE - code */
 172                 r = run_simple_cmd(testcase_path, "pre", errbuf,
 173                     sizeof(errbuf), &tr);
 174                 nopre = 0;
 175                 break;
 176
 177         case TESTCASE_CUSTOM_PRE:
 178                 /* Test case uses external and explicit PRE command */
 179                 sprintf(prepost_path, "%s/%s", prepost_dir,
 180                     testcase_get_custom_precmd(testcase));
 181
 182                 r = run_simple_cmd(prepost_path, NULL, errbuf, sizeof(errbuf),
 183                     &tr);
 184                 nopre = 0;
 185                 break;
 186
 187         default:
 188                 nopre = 1;
 189                 r = 0;
 190                 break;
 191         }
 192
 193         if (!nopre) {
 194                 if (r != 0) {
 195                         testcase_set_sys_buf(testcase, errbuf);
 196                         testcase_set_result(testcase, RESULT_PREFAIL);
 197                         goto out;
 198                 }
 199
 200                 if (tr.stdout_buf != NULL) {
 201                         testcase_set_precmd_buf(testcase, tr.stdout_buf);
 202                         free(tr.stdout_buf);
 203                 }
 204
 205                 if (tr.result != RESULT_PASS) {
 206                         testcase_set_result(testcase, RESULT_PREFAIL);
 207                         goto out;
 208                 }
 209         }
 210
 211         switch (testcase_get_type(testcase)) {
 212         case TESTCASE_TYPE_BUILDONLY:
 213                 testcase_set_result(testcase, RESULT_PASS);
 214                 testcase_set_exit_value(testcase, 0);
 215                 break;
 216
 217         case TESTCASE_TYPE_USERLAND:
 218                 /* Main testcase execution */
 219                 r = run_userland(testcase_path, testcase_get_args(testcase),
 220                     testcase_needs_setuid(testcase),
 221                     testcase_get_runas_uid(testcase),
 222                     testcase_get_timeout(testcase), 0, errbuf, sizeof(errbuf),
 223                     &tr);
 224
 225                 if (r == 0) {
 226                         testcase_set_result(testcase, tr.result);
 227                         testcase_set_exit_value(testcase, tr.exit_value);
 228                         testcase_set_signal(testcase, tr.signal);
 229
 230                         if (tr.stdout_buf != NULL) {
 231                                 testcase_set_stdout_buf(testcase, tr.stdout_buf);
 232                                 free(tr.stdout_buf);
 233                         }
 234
 235                         if (tr.stderr_buf != NULL) {
 236                                 testcase_set_stderr_buf(testcase, tr.stderr_buf);
 237                                 free(tr.stderr_buf);
 238                         }
 239                 } else {
 240                         /* driver/monitor error */
 241                         testcase_set_sys_buf(testcase, errbuf);
 242                 }
 243
 244                 break;
 245
 246         case TESTCASE_TYPE_KERNEL:
 247                 run_kernel(testcase_path, testcase);
 248                 break;
 249         }
 250
 251
 252         /* Post-execution run */
 253         switch (testcase_get_postcmd_type(testcase)) {
 254         case TESTCASE_INT_POST:
 255                 /* Test case has internal but explicit POST - code */
 256                 r = run_simple_cmd(testcase_path, "post", errbuf,
 257                     sizeof(errbuf), &tr);
 258                 nopost = 0;
 259                 break;
 260
 261         case TESTCASE_CUSTOM_POST:
 262                 /* Test case uses external and explicit POST command */
 263                 sprintf(prepost_path, "%s/%s", prepost_dir,
 264                     testcase_get_custom_postcmd(testcase));
 265
 266                 r = run_simple_cmd(prepost_path, NULL, errbuf, sizeof(errbuf),
 267                     &tr);
 268                 nopost = 0;
 269                 break;
 270
 271         default:
 272                 r = 0;
 273                 nopost = 1;
 274                 break;
 275         }
 276
 277         if (!nopost) {
 278                 if (r != 0) {
 279                         testcase_set_sys_buf(testcase, errbuf);
 280                         testcase_set_result(testcase, RESULT_POSTFAIL);
 281                         goto out;
 282                 }
 283
 284                 if (tr.stdout_buf != NULL) {
 285                         testcase_set_postcmd_buf(testcase, tr.stdout_buf);
 286                         free(tr.stdout_buf);
 287                 }
 288
 289                 if (tr.result != RESULT_PASS) {
 290                         testcase_set_result(testcase, RESULT_POSTFAIL);
 291                         goto out;
 292                 }
 293         }
 294
 295
 296
 297 out:
 298         /* clean build unless nobuild flag is set */
 299         if ((testcase_get_flags(testcase) & TESTCASE_NOBUILD) == 0) {
 300                 r = run_simple_cmd(testcase_get_make_cmd(testcase), "clean",
 301                     errbuf, sizeof(errbuf), &tr);
 302
 303                 if (tr.stdout_buf != NULL) {
 304                         testcase_set_cleanup_buf(testcase, tr.stdout_buf);
 305                         free(tr.stdout_buf);
 306                 }
 307
 308                 if (r != 0)
 309                         testcase_set_cleanup_buf(testcase, errbuf);
 310         }
 311
 312
 313         /* ... and save results */
 314         runlist_save(output_file, runlist);
 315
 316         printf("done.\n");
 317         return 0;
 318 }