drivers/dma-buf/st-dma-fence.c

   1 /* SPDX-License-Identifier: MIT */
   2
   3 /*
   4  * Copyright © 2019 Intel Corporation
   5  */
   6
   7 #include <linux/delay.h>
   8 #include <linux/dma-fence.h>
   9 #include <linux/kernel.h>
  10 #include <linux/kthread.h>
  11 #include <linux/sched/signal.h>
  12 #include <linux/slab.h>
  13 #include <linux/spinlock.h>
  14
  15 #include "selftest.h"
  16
  17 static struct kmem_cache *slab_fences;
  18
  19 static struct mock_fence {
  20         struct dma_fence base;
  21         struct spinlock lock;
  22 } *to_mock_fence(struct dma_fence *f) {
  23         return container_of(f, struct mock_fence, base);
  24 }
  25
  26 static const char *mock_name(struct dma_fence *f)
  27 {
  28         return "mock";
  29 }
  30
  31 static void mock_fence_release(struct dma_fence *f)
  32 {
  33         kmem_cache_free(slab_fences, to_mock_fence(f));
  34 }
  35
  36 struct wait_cb {
  37         struct dma_fence_cb cb;
  38         struct task_struct *task;
  39 };
  40
  41 static void mock_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
  42 {
  43         wake_up_process(container_of(cb, struct wait_cb, cb)->task);
  44 }
  45
  46 static long mock_wait(struct dma_fence *f, bool intr, long timeout)
  47 {
  48         const int state = intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
  49         struct wait_cb cb = { .task = current };
  50
  51         if (dma_fence_add_callback(f, &cb.cb, mock_wakeup))
  52                 return timeout;
  53
  54         while (timeout) {
  55                 set_current_state(state);
  56
  57                 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &f->flags))
  58                         break;
  59
  60                 if (signal_pending_state(state, current))
  61                         break;
  62
  63                 timeout = schedule_timeout(timeout);
  64         }
  65         __set_current_state(TASK_RUNNING);
  66
  67         if (!dma_fence_remove_callback(f, &cb.cb))
  68                 return timeout;
  69
  70         if (signal_pending_state(state, current))
  71                 return -ERESTARTSYS;
  72
  73         return -ETIME;
  74 }
  75
  76 static const struct dma_fence_ops mock_ops = {
  77         .get_driver_name = mock_name,
  78         .get_timeline_name = mock_name,
  79         .wait = mock_wait,
  80         .release = mock_fence_release,
  81 };
  82
  83 static struct dma_fence *mock_fence(void)
  84 {
  85         struct mock_fence *f;
  86
  87         f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
  88         if (!f)
  89                 return NULL;
  90
  91         spin_lock_init(&f->lock);
  92         dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
  93
  94         return &f->base;
  95 }
  96
  97 static int sanitycheck(void *arg)
  98 {
  99         struct dma_fence *f;
 100
 101         f = mock_fence();
 102         if (!f)
 103                 return -ENOMEM;
 104
 105         dma_fence_enable_sw_signaling(f);
 106
 107         dma_fence_signal(f);
 108         dma_fence_put(f);
 109
 110         return 0;
 111 }
 112
 113 static int test_signaling(void *arg)
 114 {
 115         struct dma_fence *f;
 116         int err = -EINVAL;
 117
 118         f = mock_fence();
 119         if (!f)
 120                 return -ENOMEM;
 121
 122         dma_fence_enable_sw_signaling(f);
 123
 124         if (dma_fence_is_signaled(f)) {
 125                 pr_err("Fence unexpectedly signaled on creation\n");
 126                 goto err_free;
 127         }
 128
 129         if (dma_fence_signal(f)) {
 130                 pr_err("Fence reported being already signaled\n");
 131                 goto err_free;
 132         }
 133
 134         if (!dma_fence_is_signaled(f)) {
 135                 pr_err("Fence not reporting signaled\n");
 136                 goto err_free;
 137         }
 138
 139         if (!dma_fence_signal(f)) {
 140                 pr_err("Fence reported not being already signaled\n");
 141                 goto err_free;
 142         }
 143
 144         err = 0;
 145 err_free:
 146         dma_fence_put(f);
 147         return err;
 148 }
 149
 150 struct simple_cb {
 151         struct dma_fence_cb cb;
 152         bool seen;
 153 };
 154
 155 static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
 156 {
 157         smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
 158 }
 159
 160 static int test_add_callback(void *arg)
 161 {
 162         struct simple_cb cb = {};
 163         struct dma_fence *f;
 164         int err = -EINVAL;
 165
 166         f = mock_fence();
 167         if (!f)
 168                 return -ENOMEM;
 169
 170         if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 171                 pr_err("Failed to add callback, fence already signaled!\n");
 172                 goto err_free;
 173         }
 174
 175         dma_fence_signal(f);
 176         if (!cb.seen) {
 177                 pr_err("Callback failed!\n");
 178                 goto err_free;
 179         }
 180
 181         err = 0;
 182 err_free:
 183         dma_fence_put(f);
 184         return err;
 185 }
 186
 187 static int test_late_add_callback(void *arg)
 188 {
 189         struct simple_cb cb = {};
 190         struct dma_fence *f;
 191         int err = -EINVAL;
 192
 193         f = mock_fence();
 194         if (!f)
 195                 return -ENOMEM;
 196
 197         dma_fence_enable_sw_signaling(f);
 198
 199         dma_fence_signal(f);
 200
 201         if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 202                 pr_err("Added callback, but fence was already signaled!\n");
 203                 goto err_free;
 204         }
 205
 206         dma_fence_signal(f);
 207         if (cb.seen) {
 208                 pr_err("Callback called after failed attachment !\n");
 209                 goto err_free;
 210         }
 211
 212         err = 0;
 213 err_free:
 214         dma_fence_put(f);
 215         return err;
 216 }
 217
 218 static int test_rm_callback(void *arg)
 219 {
 220         struct simple_cb cb = {};
 221         struct dma_fence *f;
 222         int err = -EINVAL;
 223
 224         f = mock_fence();
 225         if (!f)
 226                 return -ENOMEM;
 227
 228         if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 229                 pr_err("Failed to add callback, fence already signaled!\n");
 230                 goto err_free;
 231         }
 232
 233         if (!dma_fence_remove_callback(f, &cb.cb)) {
 234                 pr_err("Failed to remove callback!\n");
 235                 goto err_free;
 236         }
 237
 238         dma_fence_signal(f);
 239         if (cb.seen) {
 240                 pr_err("Callback still signaled after removal!\n");
 241                 goto err_free;
 242         }
 243
 244         err = 0;
 245 err_free:
 246         dma_fence_put(f);
 247         return err;
 248 }
 249
 250 static int test_late_rm_callback(void *arg)
 251 {
 252         struct simple_cb cb = {};
 253         struct dma_fence *f;
 254         int err = -EINVAL;
 255
 256         f = mock_fence();
 257         if (!f)
 258                 return -ENOMEM;
 259
 260         if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
 261                 pr_err("Failed to add callback, fence already signaled!\n");
 262                 goto err_free;
 263         }
 264
 265         dma_fence_signal(f);
 266         if (!cb.seen) {
 267                 pr_err("Callback failed!\n");
 268                 goto err_free;
 269         }
 270
 271         if (dma_fence_remove_callback(f, &cb.cb)) {
 272                 pr_err("Callback removal succeed after being executed!\n");
 273                 goto err_free;
 274         }
 275
 276         err = 0;
 277 err_free:
 278         dma_fence_put(f);
 279         return err;
 280 }
 281
 282 static int test_status(void *arg)
 283 {
 284         struct dma_fence *f;
 285         int err = -EINVAL;
 286
 287         f = mock_fence();
 288         if (!f)
 289                 return -ENOMEM;
 290
 291         dma_fence_enable_sw_signaling(f);
 292
 293         if (dma_fence_get_status(f)) {
 294                 pr_err("Fence unexpectedly has signaled status on creation\n");
 295                 goto err_free;
 296         }
 297
 298         dma_fence_signal(f);
 299         if (!dma_fence_get_status(f)) {
 300                 pr_err("Fence not reporting signaled status\n");
 301                 goto err_free;
 302         }
 303
 304         err = 0;
 305 err_free:
 306         dma_fence_put(f);
 307         return err;
 308 }
 309
 310 static int test_error(void *arg)
 311 {
 312         struct dma_fence *f;
 313         int err = -EINVAL;
 314
 315         f = mock_fence();
 316         if (!f)
 317                 return -ENOMEM;
 318
 319         dma_fence_enable_sw_signaling(f);
 320
 321         dma_fence_set_error(f, -EIO);
 322
 323         if (dma_fence_get_status(f)) {
 324                 pr_err("Fence unexpectedly has error status before signal\n");
 325                 goto err_free;
 326         }
 327
 328         dma_fence_signal(f);
 329         if (dma_fence_get_status(f) != -EIO) {
 330                 pr_err("Fence not reporting error status, got %d\n",
 331                        dma_fence_get_status(f));
 332                 goto err_free;
 333         }
 334
 335         err = 0;
 336 err_free:
 337         dma_fence_put(f);
 338         return err;
 339 }
 340
 341 static int test_wait(void *arg)
 342 {
 343         struct dma_fence *f;
 344         int err = -EINVAL;
 345
 346         f = mock_fence();
 347         if (!f)
 348                 return -ENOMEM;
 349
 350         dma_fence_enable_sw_signaling(f);
 351
 352         if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
 353                 pr_err("Wait reported complete before being signaled\n");
 354                 goto err_free;
 355         }
 356
 357         dma_fence_signal(f);
 358
 359         if (dma_fence_wait_timeout(f, false, 0) != 0) {
 360                 pr_err("Wait reported incomplete after being signaled\n");
 361                 goto err_free;
 362         }
 363
 364         err = 0;
 365 err_free:
 366         dma_fence_signal(f);
 367         dma_fence_put(f);
 368         return err;
 369 }
 370
 371 struct wait_timer {
 372         struct timer_list timer;
 373         struct dma_fence *f;
 374 };
 375
 376 static void wait_timer(struct timer_list *timer)
 377 {
 378         struct wait_timer *wt = from_timer(wt, timer, timer);
 379
 380         dma_fence_signal(wt->f);
 381 }
 382
 383 static int test_wait_timeout(void *arg)
 384 {
 385         struct wait_timer wt;
 386         int err = -EINVAL;
 387
 388         timer_setup_on_stack(&wt.timer, wait_timer, 0);
 389
 390         wt.f = mock_fence();
 391         if (!wt.f)
 392                 return -ENOMEM;
 393
 394         dma_fence_enable_sw_signaling(wt.f);
 395
 396         if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
 397                 pr_err("Wait reported complete before being signaled\n");
 398                 goto err_free;
 399         }
 400
 401         mod_timer(&wt.timer, jiffies + 1);
 402
 403         if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) {
 404                 if (timer_pending(&wt.timer)) {
 405                         pr_notice("Timer did not fire within the jiffie!\n");
 406                         err = 0; /* not our fault! */
 407                 } else {
 408                         pr_err("Wait reported incomplete after timeout\n");
 409                 }
 410                 goto err_free;
 411         }
 412
 413         err = 0;
 414 err_free:
 415         del_timer_sync(&wt.timer);
 416         destroy_timer_on_stack(&wt.timer);
 417         dma_fence_signal(wt.f);
 418         dma_fence_put(wt.f);
 419         return err;
 420 }
 421
 422 static int test_stub(void *arg)
 423 {
 424         struct dma_fence *f[64];
 425         int err = -EINVAL;
 426         int i;
 427
 428         for (i = 0; i < ARRAY_SIZE(f); i++) {
 429                 f[i] = dma_fence_get_stub();
 430                 if (!dma_fence_is_signaled(f[i])) {
 431                         pr_err("Obtained unsignaled stub fence!\n");
 432                         goto err;
 433                 }
 434         }
 435
 436         err = 0;
 437 err:
 438         while (i--)
 439                 dma_fence_put(f[i]);
 440         return err;
 441 }
 442
 443 /* Now off to the races! */
 444
 445 struct race_thread {
 446         struct dma_fence __rcu **fences;
 447         struct task_struct *task;
 448         bool before;
 449         int id;
 450 };
 451
 452 static void __wait_for_callbacks(struct dma_fence *f)
 453 {
 454         spin_lock_irq(f->lock);
 455         spin_unlock_irq(f->lock);
 456 }
 457
 458 static int thread_signal_callback(void *arg)
 459 {
 460         const struct race_thread *t = arg;
 461         unsigned long pass = 0;
 462         unsigned long miss = 0;
 463         int err = 0;
 464
 465         while (!err && !kthread_should_stop()) {
 466                 struct dma_fence *f1, *f2;
 467                 struct simple_cb cb;
 468
 469                 f1 = mock_fence();
 470                 if (!f1) {
 471                         err = -ENOMEM;
 472                         break;
 473                 }
 474
 475                 dma_fence_enable_sw_signaling(f1);
 476
 477                 rcu_assign_pointer(t->fences[t->id], f1);
 478                 smp_wmb();
 479
 480                 rcu_read_lock();
 481                 do {
 482                         f2 = dma_fence_get_rcu_safe(&t->fences[!t->id]);
 483                 } while (!f2 && !kthread_should_stop());
 484                 rcu_read_unlock();
 485
 486                 if (t->before)
 487                         dma_fence_signal(f1);
 488
 489                 smp_store_mb(cb.seen, false);
 490                 if (!f2 ||
 491                     dma_fence_add_callback(f2, &cb.cb, simple_callback)) {
 492                         miss++;
 493                         cb.seen = true;
 494                 }
 495
 496                 if (!t->before)
 497                         dma_fence_signal(f1);
 498
 499                 if (!cb.seen) {
 500                         dma_fence_wait(f2, false);
 501                         __wait_for_callbacks(f2);
 502                 }
 503
 504                 if (!READ_ONCE(cb.seen)) {
 505                         pr_err("Callback not seen on thread %d, pass %lu (%lu misses), signaling %s add_callback; fence signaled? %s\n",
 506                                t->id, pass, miss,
 507                                t->before ? "before" : "after",
 508                                dma_fence_is_signaled(f2) ? "yes" : "no");
 509                         err = -EINVAL;
 510                 }
 511
 512                 dma_fence_put(f2);
 513
 514                 rcu_assign_pointer(t->fences[t->id], NULL);
 515                 smp_wmb();
 516
 517                 dma_fence_put(f1);
 518
 519                 pass++;
 520         }
 521
 522         pr_info("%s[%d] completed %lu passes, %lu misses\n",
 523                 __func__, t->id, pass, miss);
 524         return err;
 525 }
 526
 527 static int race_signal_callback(void *arg)
 528 {
 529         struct dma_fence __rcu *f[2] = {};
 530         int ret = 0;
 531         int pass;
 532
 533         for (pass = 0; !ret && pass <= 1; pass++) {
 534                 struct race_thread t[2];
 535                 int i;
 536
 537                 for (i = 0; i < ARRAY_SIZE(t); i++) {
 538                         t[i].fences = f;
 539                         t[i].id = i;
 540                         t[i].before = pass;
 541                         t[i].task = kthread_run(thread_signal_callback, &t[i],
 542                                                 "dma-fence:%d", i);
 543                         get_task_struct(t[i].task);
 544                 }
 545
 546                 msleep(50);
 547
 548                 for (i = 0; i < ARRAY_SIZE(t); i++) {
 549                         int err;
 550
 551                         err = kthread_stop(t[i].task);
 552                         if (err && !ret)
 553                                 ret = err;
 554
 555                         put_task_struct(t[i].task);
 556                 }
 557         }
 558
 559         return ret;
 560 }
 561
 562 int dma_fence(void)
 563 {
 564         static const struct subtest tests[] = {
 565                 SUBTEST(sanitycheck),
 566                 SUBTEST(test_signaling),
 567                 SUBTEST(test_add_callback),
 568                 SUBTEST(test_late_add_callback),
 569                 SUBTEST(test_rm_callback),
 570                 SUBTEST(test_late_rm_callback),
 571                 SUBTEST(test_status),
 572                 SUBTEST(test_error),
 573                 SUBTEST(test_wait),
 574                 SUBTEST(test_wait_timeout),
 575                 SUBTEST(test_stub),
 576                 SUBTEST(race_signal_callback),
 577         };
 578         int ret;
 579
 580         pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
 581
 582         slab_fences = KMEM_CACHE(mock_fence,
 583                                  SLAB_TYPESAFE_BY_RCU |
 584                                  SLAB_HWCACHE_ALIGN);
 585         if (!slab_fences)
 586                 return -ENOMEM;
 587
 588         ret = subtests(tests, NULL);
 589
 590         kmem_cache_destroy(slab_fences);
 591
 592         return ret;
 593 }