kernel/rcu/refscale.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 //
   3 // Scalability test comparing RCU vs other mechanisms
   4 // for acquiring references on objects.
   5 //
   6 // Copyright (C) Google, 2020.
   7 //
   8 // Author: Joel Fernandes <joel@joelfernandes.org>
   9
  10 #define pr_fmt(fmt) fmt
  11
  12 #include <linux/atomic.h>
  13 #include <linux/bitops.h>
  14 #include <linux/completion.h>
  15 #include <linux/cpu.h>
  16 #include <linux/delay.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/kthread.h>
  21 #include <linux/kernel.h>
  22 #include <linux/mm.h>
  23 #include <linux/module.h>
  24 #include <linux/moduleparam.h>
  25 #include <linux/notifier.h>
  26 #include <linux/percpu.h>
  27 #include <linux/rcupdate.h>
  28 #include <linux/rcupdate_trace.h>
  29 #include <linux/reboot.h>
  30 #include <linux/sched.h>
  31 #include <linux/seq_buf.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/smp.h>
  34 #include <linux/stat.h>
  35 #include <linux/srcu.h>
  36 #include <linux/slab.h>
  37 #include <linux/torture.h>
  38 #include <linux/types.h>
  39
  40 #include "rcu.h"
  41
  42 #define SCALE_FLAG "-ref-scale: "
  43
  44 #define SCALEOUT(s, x...) \
  45         pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
  46
  47 #define VERBOSE_SCALEOUT(s, x...) \
  48         do { \
  49                 if (verbose) \
  50                         pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
  51         } while (0)
  52
  53 static atomic_t verbose_batch_ctr;
  54
  55 #define VERBOSE_SCALEOUT_BATCH(s, x...)                                                 \
  56 do {                                                                                    \
  57         if (verbose &&                                                                  \
  58             (verbose_batched <= 0 ||                                                    \
  59              !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {             \
  60                 schedule_timeout_uninterruptible(1);                                    \
  61                 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x);                     \
  62         }                                                                               \
  63 } while (0)
  64
  65 #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
  66
  67 MODULE_DESCRIPTION("Scalability test for object reference mechanisms");
  68 MODULE_LICENSE("GPL");
  69 MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
  70
  71 static char *scale_type = "rcu";
  72 module_param(scale_type, charp, 0444);
  73 MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
  74
  75 torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
  76 torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
  77
  78 // Wait until there are multiple CPUs before starting test.
  79 torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
  80               "Holdoff time before test start (s)");
  81 // Number of typesafe_lookup structures, that is, the degree of concurrency.
  82 torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");
  83 // Number of loops per experiment, all readers execute operations concurrently.
  84 torture_param(long, loops, 10000, "Number of loops per experiment.");
  85 // Number of readers, with -1 defaulting to about 75% of the CPUs.
  86 torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
  87 // Number of runs.
  88 torture_param(int, nruns, 30, "Number of experiments to run.");
  89 // Reader delay in nanoseconds, 0 for no delay.
  90 torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
  91
  92 #ifdef MODULE
  93 # define REFSCALE_SHUTDOWN 0
  94 #else
  95 # define REFSCALE_SHUTDOWN 1
  96 #endif
  97
  98 torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
  99               "Shutdown at end of scalability tests.");
 100
 101 struct reader_task {
 102         struct task_struct *task;
 103         int start_reader;
 104         wait_queue_head_t wq;
 105         u64 last_duration_ns;
 106 };
 107
 108 static struct task_struct *shutdown_task;
 109 static wait_queue_head_t shutdown_wq;
 110
 111 static struct task_struct *main_task;
 112 static wait_queue_head_t main_wq;
 113 static int shutdown_start;
 114
 115 static struct reader_task *reader_tasks;
 116
 117 // Number of readers that are part of the current experiment.
 118 static atomic_t nreaders_exp;
 119
 120 // Use to wait for all threads to start.
 121 static atomic_t n_init;
 122 static atomic_t n_started;
 123 static atomic_t n_warmedup;
 124 static atomic_t n_cooleddown;
 125
 126 // Track which experiment is currently running.
 127 static int exp_idx;
 128
 129 // Operations vector for selecting different types of tests.
 130 struct ref_scale_ops {
 131         bool (*init)(void);
 132         void (*cleanup)(void);
 133         void (*readsection)(const int nloops);
 134         void (*delaysection)(const int nloops, const int udl, const int ndl);
 135         const char *name;
 136 };
 137
 138 static const struct ref_scale_ops *cur_ops;
 139
 140 static void un_delay(const int udl, const int ndl)
 141 {
 142         if (udl)
 143                 udelay(udl);
 144         if (ndl)
 145                 ndelay(ndl);
 146 }
 147
 148 static void ref_rcu_read_section(const int nloops)
 149 {
 150         int i;
 151
 152         for (i = nloops; i >= 0; i--) {
 153                 rcu_read_lock();
 154                 rcu_read_unlock();
 155         }
 156 }
 157
 158 static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
 159 {
 160         int i;
 161
 162         for (i = nloops; i >= 0; i--) {
 163                 rcu_read_lock();
 164                 un_delay(udl, ndl);
 165                 rcu_read_unlock();
 166         }
 167 }
 168
 169 static bool rcu_sync_scale_init(void)
 170 {
 171         return true;
 172 }
 173
 174 static const struct ref_scale_ops rcu_ops = {
 175         .init           = rcu_sync_scale_init,
 176         .readsection    = ref_rcu_read_section,
 177         .delaysection   = ref_rcu_delay_section,
 178         .name           = "rcu"
 179 };
 180
 181 // Definitions for SRCU ref scale testing.
 182 DEFINE_STATIC_SRCU(srcu_refctl_scale);
 183 static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
 184
 185 static void srcu_ref_scale_read_section(const int nloops)
 186 {
 187         int i;
 188         int idx;
 189
 190         for (i = nloops; i >= 0; i--) {
 191                 idx = srcu_read_lock(srcu_ctlp);
 192                 srcu_read_unlock(srcu_ctlp, idx);
 193         }
 194 }
 195
 196 static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 197 {
 198         int i;
 199         int idx;
 200
 201         for (i = nloops; i >= 0; i--) {
 202                 idx = srcu_read_lock(srcu_ctlp);
 203                 un_delay(udl, ndl);
 204                 srcu_read_unlock(srcu_ctlp, idx);
 205         }
 206 }
 207
 208 static const struct ref_scale_ops srcu_ops = {
 209         .init           = rcu_sync_scale_init,
 210         .readsection    = srcu_ref_scale_read_section,
 211         .delaysection   = srcu_ref_scale_delay_section,
 212         .name           = "srcu"
 213 };
 214
 215 #ifdef CONFIG_TASKS_RCU
 216
 217 // Definitions for RCU Tasks ref scale testing: Empty read markers.
 218 // These definitions also work for RCU Rude readers.
 219 static void rcu_tasks_ref_scale_read_section(const int nloops)
 220 {
 221         int i;
 222
 223         for (i = nloops; i >= 0; i--)
 224                 continue;
 225 }
 226
 227 static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 228 {
 229         int i;
 230
 231         for (i = nloops; i >= 0; i--)
 232                 un_delay(udl, ndl);
 233 }
 234
 235 static const struct ref_scale_ops rcu_tasks_ops = {
 236         .init           = rcu_sync_scale_init,
 237         .readsection    = rcu_tasks_ref_scale_read_section,
 238         .delaysection   = rcu_tasks_ref_scale_delay_section,
 239         .name           = "rcu-tasks"
 240 };
 241
 242 #define RCU_TASKS_OPS &rcu_tasks_ops,
 243
 244 #else // #ifdef CONFIG_TASKS_RCU
 245
 246 #define RCU_TASKS_OPS
 247
 248 #endif // #else // #ifdef CONFIG_TASKS_RCU
 249
 250 #ifdef CONFIG_TASKS_TRACE_RCU
 251
 252 // Definitions for RCU Tasks Trace ref scale testing.
 253 static void rcu_trace_ref_scale_read_section(const int nloops)
 254 {
 255         int i;
 256
 257         for (i = nloops; i >= 0; i--) {
 258                 rcu_read_lock_trace();
 259                 rcu_read_unlock_trace();
 260         }
 261 }
 262
 263 static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
 264 {
 265         int i;
 266
 267         for (i = nloops; i >= 0; i--) {
 268                 rcu_read_lock_trace();
 269                 un_delay(udl, ndl);
 270                 rcu_read_unlock_trace();
 271         }
 272 }
 273
 274 static const struct ref_scale_ops rcu_trace_ops = {
 275         .init           = rcu_sync_scale_init,
 276         .readsection    = rcu_trace_ref_scale_read_section,
 277         .delaysection   = rcu_trace_ref_scale_delay_section,
 278         .name           = "rcu-trace"
 279 };
 280
 281 #define RCU_TRACE_OPS &rcu_trace_ops,
 282
 283 #else // #ifdef CONFIG_TASKS_TRACE_RCU
 284
 285 #define RCU_TRACE_OPS
 286
 287 #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
 288
 289 // Definitions for reference count
 290 static atomic_t refcnt;
 291
 292 static void ref_refcnt_section(const int nloops)
 293 {
 294         int i;
 295
 296         for (i = nloops; i >= 0; i--) {
 297                 atomic_inc(&refcnt);
 298                 atomic_dec(&refcnt);
 299         }
 300 }
 301
 302 static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
 303 {
 304         int i;
 305
 306         for (i = nloops; i >= 0; i--) {
 307                 atomic_inc(&refcnt);
 308                 un_delay(udl, ndl);
 309                 atomic_dec(&refcnt);
 310         }
 311 }
 312
 313 static const struct ref_scale_ops refcnt_ops = {
 314         .init           = rcu_sync_scale_init,
 315         .readsection    = ref_refcnt_section,
 316         .delaysection   = ref_refcnt_delay_section,
 317         .name           = "refcnt"
 318 };
 319
 320 // Definitions for rwlock
 321 static rwlock_t test_rwlock;
 322
 323 static bool ref_rwlock_init(void)
 324 {
 325         rwlock_init(&test_rwlock);
 326         return true;
 327 }
 328
 329 static void ref_rwlock_section(const int nloops)
 330 {
 331         int i;
 332
 333         for (i = nloops; i >= 0; i--) {
 334                 read_lock(&test_rwlock);
 335                 read_unlock(&test_rwlock);
 336         }
 337 }
 338
 339 static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
 340 {
 341         int i;
 342
 343         for (i = nloops; i >= 0; i--) {
 344                 read_lock(&test_rwlock);
 345                 un_delay(udl, ndl);
 346                 read_unlock(&test_rwlock);
 347         }
 348 }
 349
 350 static const struct ref_scale_ops rwlock_ops = {
 351         .init           = ref_rwlock_init,
 352         .readsection    = ref_rwlock_section,
 353         .delaysection   = ref_rwlock_delay_section,
 354         .name           = "rwlock"
 355 };
 356
 357 // Definitions for rwsem
 358 static struct rw_semaphore test_rwsem;
 359
 360 static bool ref_rwsem_init(void)
 361 {
 362         init_rwsem(&test_rwsem);
 363         return true;
 364 }
 365
 366 static void ref_rwsem_section(const int nloops)
 367 {
 368         int i;
 369
 370         for (i = nloops; i >= 0; i--) {
 371                 down_read(&test_rwsem);
 372                 up_read(&test_rwsem);
 373         }
 374 }
 375
 376 static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
 377 {
 378         int i;
 379
 380         for (i = nloops; i >= 0; i--) {
 381                 down_read(&test_rwsem);
 382                 un_delay(udl, ndl);
 383                 up_read(&test_rwsem);
 384         }
 385 }
 386
 387 static const struct ref_scale_ops rwsem_ops = {
 388         .init           = ref_rwsem_init,
 389         .readsection    = ref_rwsem_section,
 390         .delaysection   = ref_rwsem_delay_section,
 391         .name           = "rwsem"
 392 };
 393
 394 // Definitions for global spinlock
 395 static DEFINE_RAW_SPINLOCK(test_lock);
 396
 397 static void ref_lock_section(const int nloops)
 398 {
 399         int i;
 400
 401         preempt_disable();
 402         for (i = nloops; i >= 0; i--) {
 403                 raw_spin_lock(&test_lock);
 404                 raw_spin_unlock(&test_lock);
 405         }
 406         preempt_enable();
 407 }
 408
 409 static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
 410 {
 411         int i;
 412
 413         preempt_disable();
 414         for (i = nloops; i >= 0; i--) {
 415                 raw_spin_lock(&test_lock);
 416                 un_delay(udl, ndl);
 417                 raw_spin_unlock(&test_lock);
 418         }
 419         preempt_enable();
 420 }
 421
 422 static const struct ref_scale_ops lock_ops = {
 423         .readsection    = ref_lock_section,
 424         .delaysection   = ref_lock_delay_section,
 425         .name           = "lock"
 426 };
 427
 428 // Definitions for global irq-save spinlock
 429
 430 static void ref_lock_irq_section(const int nloops)
 431 {
 432         unsigned long flags;
 433         int i;
 434
 435         preempt_disable();
 436         for (i = nloops; i >= 0; i--) {
 437                 raw_spin_lock_irqsave(&test_lock, flags);
 438                 raw_spin_unlock_irqrestore(&test_lock, flags);
 439         }
 440         preempt_enable();
 441 }
 442
 443 static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
 444 {
 445         unsigned long flags;
 446         int i;
 447
 448         preempt_disable();
 449         for (i = nloops; i >= 0; i--) {
 450                 raw_spin_lock_irqsave(&test_lock, flags);
 451                 un_delay(udl, ndl);
 452                 raw_spin_unlock_irqrestore(&test_lock, flags);
 453         }
 454         preempt_enable();
 455 }
 456
 457 static const struct ref_scale_ops lock_irq_ops = {
 458         .readsection    = ref_lock_irq_section,
 459         .delaysection   = ref_lock_irq_delay_section,
 460         .name           = "lock-irq"
 461 };
 462
 463 // Definitions acquire-release.
 464 static DEFINE_PER_CPU(unsigned long, test_acqrel);
 465
 466 static void ref_acqrel_section(const int nloops)
 467 {
 468         unsigned long x;
 469         int i;
 470
 471         preempt_disable();
 472         for (i = nloops; i >= 0; i--) {
 473                 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
 474                 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
 475         }
 476         preempt_enable();
 477 }
 478
 479 static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
 480 {
 481         unsigned long x;
 482         int i;
 483
 484         preempt_disable();
 485         for (i = nloops; i >= 0; i--) {
 486                 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
 487                 un_delay(udl, ndl);
 488                 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
 489         }
 490         preempt_enable();
 491 }
 492
 493 static const struct ref_scale_ops acqrel_ops = {
 494         .readsection    = ref_acqrel_section,
 495         .delaysection   = ref_acqrel_delay_section,
 496         .name           = "acqrel"
 497 };
 498
 499 static volatile u64 stopopts;
 500
 501 static void ref_clock_section(const int nloops)
 502 {
 503         u64 x = 0;
 504         int i;
 505
 506         preempt_disable();
 507         for (i = nloops; i >= 0; i--)
 508                 x += ktime_get_real_fast_ns();
 509         preempt_enable();
 510         stopopts = x;
 511 }
 512
 513 static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
 514 {
 515         u64 x = 0;
 516         int i;
 517
 518         preempt_disable();
 519         for (i = nloops; i >= 0; i--) {
 520                 x += ktime_get_real_fast_ns();
 521                 un_delay(udl, ndl);
 522         }
 523         preempt_enable();
 524         stopopts = x;
 525 }
 526
 527 static const struct ref_scale_ops clock_ops = {
 528         .readsection    = ref_clock_section,
 529         .delaysection   = ref_clock_delay_section,
 530         .name           = "clock"
 531 };
 532
 533 static void ref_jiffies_section(const int nloops)
 534 {
 535         u64 x = 0;
 536         int i;
 537
 538         preempt_disable();
 539         for (i = nloops; i >= 0; i--)
 540                 x += jiffies;
 541         preempt_enable();
 542         stopopts = x;
 543 }
 544
 545 static void ref_jiffies_delay_section(const int nloops, const int udl, const int ndl)
 546 {
 547         u64 x = 0;
 548         int i;
 549
 550         preempt_disable();
 551         for (i = nloops; i >= 0; i--) {
 552                 x += jiffies;
 553                 un_delay(udl, ndl);
 554         }
 555         preempt_enable();
 556         stopopts = x;
 557 }
 558
 559 static const struct ref_scale_ops jiffies_ops = {
 560         .readsection    = ref_jiffies_section,
 561         .delaysection   = ref_jiffies_delay_section,
 562         .name           = "jiffies"
 563 };
 564
 565 ////////////////////////////////////////////////////////////////////////
 566 //
 567 // Methods leveraging SLAB_TYPESAFE_BY_RCU.
 568 //
 569
 570 // Item to look up in a typesafe manner.  Array of pointers to these.
 571 struct refscale_typesafe {
 572         atomic_t rts_refctr;  // Used by all flavors
 573         spinlock_t rts_lock;
 574         seqlock_t rts_seqlock;
 575         unsigned int a;
 576         unsigned int b;
 577 };
 578
 579 static struct kmem_cache *typesafe_kmem_cachep;
 580 static struct refscale_typesafe **rtsarray;
 581 static long rtsarray_size;
 582 static DEFINE_TORTURE_RANDOM_PERCPU(refscale_rand);
 583 static bool (*rts_acquire)(struct refscale_typesafe *rtsp, unsigned int *start);
 584 static bool (*rts_release)(struct refscale_typesafe *rtsp, unsigned int start);
 585
 586 // Conditionally acquire an explicit in-structure reference count.
 587 static bool typesafe_ref_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 588 {
 589         return atomic_inc_not_zero(&rtsp->rts_refctr);
 590 }
 591
 592 // Unconditionally release an explicit in-structure reference count.
 593 static bool typesafe_ref_release(struct refscale_typesafe *rtsp, unsigned int start)
 594 {
 595         if (!atomic_dec_return(&rtsp->rts_refctr)) {
 596                 WRITE_ONCE(rtsp->a, rtsp->a + 1);
 597                 kmem_cache_free(typesafe_kmem_cachep, rtsp);
 598         }
 599         return true;
 600 }
 601
 602 // Unconditionally acquire an explicit in-structure spinlock.
 603 static bool typesafe_lock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 604 {
 605         spin_lock(&rtsp->rts_lock);
 606         return true;
 607 }
 608
 609 // Unconditionally release an explicit in-structure spinlock.
 610 static bool typesafe_lock_release(struct refscale_typesafe *rtsp, unsigned int start)
 611 {
 612         spin_unlock(&rtsp->rts_lock);
 613         return true;
 614 }
 615
 616 // Unconditionally acquire an explicit in-structure sequence lock.
 617 static bool typesafe_seqlock_acquire(struct refscale_typesafe *rtsp, unsigned int *start)
 618 {
 619         *start = read_seqbegin(&rtsp->rts_seqlock);
 620         return true;
 621 }
 622
 623 // Conditionally release an explicit in-structure sequence lock.  Return
 624 // true if this release was successful, that is, if no retry is required.
 625 static bool typesafe_seqlock_release(struct refscale_typesafe *rtsp, unsigned int start)
 626 {
 627         return !read_seqretry(&rtsp->rts_seqlock, start);
 628 }
 629
 630 // Do a read-side critical section with the specified delay in
 631 // microseconds and nanoseconds inserted so as to increase probability
 632 // of failure.
 633 static void typesafe_delay_section(const int nloops, const int udl, const int ndl)
 634 {
 635         unsigned int a;
 636         unsigned int b;
 637         int i;
 638         long idx;
 639         struct refscale_typesafe *rtsp;
 640         unsigned int start;
 641
 642         for (i = nloops; i >= 0; i--) {
 643                 preempt_disable();
 644                 idx = torture_random(this_cpu_ptr(&refscale_rand)) % rtsarray_size;
 645                 preempt_enable();
 646 retry:
 647                 rcu_read_lock();
 648                 rtsp = rcu_dereference(rtsarray[idx]);
 649                 a = READ_ONCE(rtsp->a);
 650                 if (!rts_acquire(rtsp, &start)) {
 651                         rcu_read_unlock();
 652                         goto retry;
 653                 }
 654                 if (a != READ_ONCE(rtsp->a)) {
 655                         (void)rts_release(rtsp, start);
 656                         rcu_read_unlock();
 657                         goto retry;
 658                 }
 659                 un_delay(udl, ndl);
 660                 b = READ_ONCE(rtsp->a);
 661                 // Remember, seqlock read-side release can fail.
 662                 if (!rts_release(rtsp, start)) {
 663                         rcu_read_unlock();
 664                         goto retry;
 665                 }
 666                 WARN_ONCE(a != b, "Re-read of ->a changed from %u to %u.\n", a, b);
 667                 b = rtsp->b;
 668                 rcu_read_unlock();
 669                 WARN_ON_ONCE(a * a != b);
 670         }
 671 }
 672
 673 // Because the acquisition and release methods are expensive, there
 674 // is no point in optimizing away the un_delay() function's two checks.
 675 // Thus simply define typesafe_read_section() as a simple wrapper around
 676 // typesafe_delay_section().
 677 static void typesafe_read_section(const int nloops)
 678 {
 679         typesafe_delay_section(nloops, 0, 0);
 680 }
 681
 682 // Allocate and initialize one refscale_typesafe structure.
 683 static struct refscale_typesafe *typesafe_alloc_one(void)
 684 {
 685         struct refscale_typesafe *rtsp;
 686
 687         rtsp = kmem_cache_alloc(typesafe_kmem_cachep, GFP_KERNEL);
 688         if (!rtsp)
 689                 return NULL;
 690         atomic_set(&rtsp->rts_refctr, 1);
 691         WRITE_ONCE(rtsp->a, rtsp->a + 1);
 692         WRITE_ONCE(rtsp->b, rtsp->a * rtsp->a);
 693         return rtsp;
 694 }
 695
 696 // Slab-allocator constructor for refscale_typesafe structures created
 697 // out of a new slab of system memory.
 698 static void refscale_typesafe_ctor(void *rtsp_in)
 699 {
 700         struct refscale_typesafe *rtsp = rtsp_in;
 701
 702         spin_lock_init(&rtsp->rts_lock);
 703         seqlock_init(&rtsp->rts_seqlock);
 704         preempt_disable();
 705         rtsp->a = torture_random(this_cpu_ptr(&refscale_rand));
 706         preempt_enable();
 707 }
 708
 709 static const struct ref_scale_ops typesafe_ref_ops;
 710 static const struct ref_scale_ops typesafe_lock_ops;
 711 static const struct ref_scale_ops typesafe_seqlock_ops;
 712
 713 // Initialize for a typesafe test.
 714 static bool typesafe_init(void)
 715 {
 716         long idx;
 717         long si = lookup_instances;
 718
 719         typesafe_kmem_cachep = kmem_cache_create("refscale_typesafe",
 720                                                  sizeof(struct refscale_typesafe), sizeof(void *),
 721                                                  SLAB_TYPESAFE_BY_RCU, refscale_typesafe_ctor);
 722         if (!typesafe_kmem_cachep)
 723                 return false;
 724         if (si < 0)
 725                 si = -si * nr_cpu_ids;
 726         else if (si == 0)
 727                 si = nr_cpu_ids;
 728         rtsarray_size = si;
 729         rtsarray = kcalloc(si, sizeof(*rtsarray), GFP_KERNEL);
 730         if (!rtsarray)
 731                 return false;
 732         for (idx = 0; idx < rtsarray_size; idx++) {
 733                 rtsarray[idx] = typesafe_alloc_one();
 734                 if (!rtsarray[idx])
 735                         return false;
 736         }
 737         if (cur_ops == &typesafe_ref_ops) {
 738                 rts_acquire = typesafe_ref_acquire;
 739                 rts_release = typesafe_ref_release;
 740         } else if (cur_ops == &typesafe_lock_ops) {
 741                 rts_acquire = typesafe_lock_acquire;
 742                 rts_release = typesafe_lock_release;
 743         } else if (cur_ops == &typesafe_seqlock_ops) {
 744                 rts_acquire = typesafe_seqlock_acquire;
 745                 rts_release = typesafe_seqlock_release;
 746         } else {
 747                 WARN_ON_ONCE(1);
 748                 return false;
 749         }
 750         return true;
 751 }
 752
 753 // Clean up after a typesafe test.
 754 static void typesafe_cleanup(void)
 755 {
 756         long idx;
 757
 758         if (rtsarray) {
 759                 for (idx = 0; idx < rtsarray_size; idx++)
 760                         kmem_cache_free(typesafe_kmem_cachep, rtsarray[idx]);
 761                 kfree(rtsarray);
 762                 rtsarray = NULL;
 763                 rtsarray_size = 0;
 764         }
 765         kmem_cache_destroy(typesafe_kmem_cachep);
 766         typesafe_kmem_cachep = NULL;
 767         rts_acquire = NULL;
 768         rts_release = NULL;
 769 }
 770
 771 // The typesafe_init() function distinguishes these structures by address.
 772 static const struct ref_scale_ops typesafe_ref_ops = {
 773         .init           = typesafe_init,
 774         .cleanup        = typesafe_cleanup,
 775         .readsection    = typesafe_read_section,
 776         .delaysection   = typesafe_delay_section,
 777         .name           = "typesafe_ref"
 778 };
 779
 780 static const struct ref_scale_ops typesafe_lock_ops = {
 781         .init           = typesafe_init,
 782         .cleanup        = typesafe_cleanup,
 783         .readsection    = typesafe_read_section,
 784         .delaysection   = typesafe_delay_section,
 785         .name           = "typesafe_lock"
 786 };
 787
 788 static const struct ref_scale_ops typesafe_seqlock_ops = {
 789         .init           = typesafe_init,
 790         .cleanup        = typesafe_cleanup,
 791         .readsection    = typesafe_read_section,
 792         .delaysection   = typesafe_delay_section,
 793         .name           = "typesafe_seqlock"
 794 };
 795
 796 static void rcu_scale_one_reader(void)
 797 {
 798         if (readdelay <= 0)
 799                 cur_ops->readsection(loops);
 800         else
 801                 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
 802 }
 803
 804 // Reader kthread.  Repeatedly does empty RCU read-side
 805 // critical section, minimizing update-side interference.
 806 static int
 807 ref_scale_reader(void *arg)
 808 {
 809         unsigned long flags;
 810         long me = (long)arg;
 811         struct reader_task *rt = &(reader_tasks[me]);
 812         u64 start;
 813         s64 duration;
 814
 815         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
 816         WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
 817         set_user_nice(current, MAX_NICE);
 818         atomic_inc(&n_init);
 819         if (holdoff)
 820                 schedule_timeout_interruptible(holdoff * HZ);
 821 repeat:
 822         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
 823
 824         // Wait for signal that this reader can start.
 825         wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
 826                            torture_must_stop());
 827
 828         if (torture_must_stop())
 829                 goto end;
 830
 831         // Make sure that the CPU is affinitized appropriately during testing.
 832         WARN_ON_ONCE(raw_smp_processor_id() != me);
 833
 834         WRITE_ONCE(rt->start_reader, 0);
 835         if (!atomic_dec_return(&n_started))
 836                 while (atomic_read_acquire(&n_started))
 837                         cpu_relax();
 838
 839         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
 840
 841
 842         // To reduce noise, do an initial cache-warming invocation, check
 843         // in, and then keep warming until everyone has checked in.
 844         rcu_scale_one_reader();
 845         if (!atomic_dec_return(&n_warmedup))
 846                 while (atomic_read_acquire(&n_warmedup))
 847                         rcu_scale_one_reader();
 848         // Also keep interrupts disabled.  This also has the effect
 849         // of preventing entries into slow path for rcu_read_unlock().
 850         local_irq_save(flags);
 851         start = ktime_get_mono_fast_ns();
 852
 853         rcu_scale_one_reader();
 854
 855         duration = ktime_get_mono_fast_ns() - start;
 856         local_irq_restore(flags);
 857
 858         rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
 859         // To reduce runtime-skew noise, do maintain-load invocations until
 860         // everyone is done.
 861         if (!atomic_dec_return(&n_cooleddown))
 862                 while (atomic_read_acquire(&n_cooleddown))
 863                         rcu_scale_one_reader();
 864
 865         if (atomic_dec_and_test(&nreaders_exp))
 866                 wake_up(&main_wq);
 867
 868         VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
 869                                 me, exp_idx, atomic_read(&nreaders_exp));
 870
 871         if (!torture_must_stop())
 872                 goto repeat;
 873 end:
 874         torture_kthread_stopping("ref_scale_reader");
 875         return 0;
 876 }
 877
 878 static void reset_readers(void)
 879 {
 880         int i;
 881         struct reader_task *rt;
 882
 883         for (i = 0; i < nreaders; i++) {
 884                 rt = &(reader_tasks[i]);
 885
 886                 rt->last_duration_ns = 0;
 887         }
 888 }
 889
 890 // Print the results of each reader and return the sum of all their durations.
 891 static u64 process_durations(int n)
 892 {
 893         int i;
 894         struct reader_task *rt;
 895         struct seq_buf s;
 896         char *buf;
 897         u64 sum = 0;
 898
 899         buf = kmalloc(800 + 64, GFP_KERNEL);
 900         if (!buf)
 901                 return 0;
 902         seq_buf_init(&s, buf, 800 + 64);
 903
 904         seq_buf_printf(&s, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
 905                        exp_idx);
 906
 907         for (i = 0; i < n && !torture_must_stop(); i++) {
 908                 rt = &(reader_tasks[i]);
 909
 910                 if (i % 5 == 0)
 911                         seq_buf_putc(&s, '\n');
 912
 913                 if (seq_buf_used(&s) >= 800) {
 914                         pr_alert("%s", seq_buf_str(&s));
 915                         seq_buf_clear(&s);
 916                 }
 917
 918                 seq_buf_printf(&s, "%d: %llu\t", i, rt->last_duration_ns);
 919
 920                 sum += rt->last_duration_ns;
 921         }
 922         pr_alert("%s\n", seq_buf_str(&s));
 923
 924         kfree(buf);
 925         return sum;
 926 }
 927
 928 // The main_func is the main orchestrator, it performs a bunch of
 929 // experiments.  For every experiment, it orders all the readers
 930 // involved to start and waits for them to finish the experiment. It
 931 // then reads their timestamps and starts the next experiment. Each
 932 // experiment progresses from 1 concurrent reader to N of them at which
 933 // point all the timestamps are printed.
 934 static int main_func(void *arg)
 935 {
 936         int exp, r;
 937         char buf1[64];
 938         char *buf;
 939         u64 *result_avg;
 940
 941         set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
 942         set_user_nice(current, MAX_NICE);
 943
 944         VERBOSE_SCALEOUT("main_func task started");
 945         result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
 946         buf = kzalloc(800 + 64, GFP_KERNEL);
 947         if (!result_avg || !buf) {
 948                 SCALEOUT_ERRSTRING("out of memory");
 949                 goto oom_exit;
 950         }
 951         if (holdoff)
 952                 schedule_timeout_interruptible(holdoff * HZ);
 953
 954         // Wait for all threads to start.
 955         atomic_inc(&n_init);
 956         while (atomic_read(&n_init) < nreaders + 1)
 957                 schedule_timeout_uninterruptible(1);
 958
 959         // Start exp readers up per experiment
 960         for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
 961                 if (torture_must_stop())
 962                         goto end;
 963
 964                 reset_readers();
 965                 atomic_set(&nreaders_exp, nreaders);
 966                 atomic_set(&n_started, nreaders);
 967                 atomic_set(&n_warmedup, nreaders);
 968                 atomic_set(&n_cooleddown, nreaders);
 969
 970                 exp_idx = exp;
 971
 972                 for (r = 0; r < nreaders; r++) {
 973                         smp_store_release(&reader_tasks[r].start_reader, 1);
 974                         wake_up(&reader_tasks[r].wq);
 975                 }
 976
 977                 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
 978                                 nreaders);
 979
 980                 wait_event(main_wq,
 981                            !atomic_read(&nreaders_exp) || torture_must_stop());
 982
 983                 VERBOSE_SCALEOUT("main_func: experiment ended");
 984
 985                 if (torture_must_stop())
 986                         goto end;
 987
 988                 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
 989         }
 990
 991         // Print the average of all experiments
 992         SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
 993
 994         pr_alert("Runs\tTime(ns)\n");
 995         for (exp = 0; exp < nruns; exp++) {
 996                 u64 avg;
 997                 u32 rem;
 998
 999                 avg = div_u64_rem(result_avg[exp], 1000, &rem);
1000                 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
1001                 strcat(buf, buf1);
1002                 if (strlen(buf) >= 800) {
1003                         pr_alert("%s", buf);
1004                         buf[0] = 0;
1005                 }
1006         }
1007
1008         pr_alert("%s", buf);
1009
1010 oom_exit:
1011         // This will shutdown everything including us.
1012         if (shutdown) {
1013                 shutdown_start = 1;
1014                 wake_up(&shutdown_wq);
1015         }
1016
1017         // Wait for torture to stop us
1018         while (!torture_must_stop())
1019                 schedule_timeout_uninterruptible(1);
1020
1021 end:
1022         torture_kthread_stopping("main_func");
1023         kfree(result_avg);
1024         kfree(buf);
1025         return 0;
1026 }
1027
1028 static void
1029 ref_scale_print_module_parms(const struct ref_scale_ops *cur_ops, const char *tag)
1030 {
1031         pr_alert("%s" SCALE_FLAG
1032                  "--- %s:  verbose=%d verbose_batched=%d shutdown=%d holdoff=%d lookup_instances=%ld loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
1033                  verbose, verbose_batched, shutdown, holdoff, lookup_instances, loops, nreaders, nruns, readdelay);
1034 }
1035
1036 static void
1037 ref_scale_cleanup(void)
1038 {
1039         int i;
1040
1041         if (torture_cleanup_begin())
1042                 return;
1043
1044         if (!cur_ops) {
1045                 torture_cleanup_end();
1046                 return;
1047         }
1048
1049         if (reader_tasks) {
1050                 for (i = 0; i < nreaders; i++)
1051                         torture_stop_kthread("ref_scale_reader",
1052                                              reader_tasks[i].task);
1053         }
1054         kfree(reader_tasks);
1055
1056         torture_stop_kthread("main_task", main_task);
1057         kfree(main_task);
1058
1059         // Do scale-type-specific cleanup operations.
1060         if (cur_ops->cleanup != NULL)
1061                 cur_ops->cleanup();
1062
1063         torture_cleanup_end();
1064 }
1065
1066 // Shutdown kthread.  Just waits to be awakened, then shuts down system.
1067 static int
1068 ref_scale_shutdown(void *arg)
1069 {
1070         wait_event_idle(shutdown_wq, shutdown_start);
1071
1072         smp_mb(); // Wake before output.
1073         ref_scale_cleanup();
1074         kernel_power_off();
1075
1076         return -EINVAL;
1077 }
1078
1079 static int __init
1080 ref_scale_init(void)
1081 {
1082         long i;
1083         int firsterr = 0;
1084         static const struct ref_scale_ops *scale_ops[] = {
1085                 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
1086                 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, &jiffies_ops,
1087                 &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
1088         };
1089
1090         if (!torture_init_begin(scale_type, verbose))
1091                 return -EBUSY;
1092
1093         for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
1094                 cur_ops = scale_ops[i];
1095                 if (strcmp(scale_type, cur_ops->name) == 0)
1096                         break;
1097         }
1098         if (i == ARRAY_SIZE(scale_ops)) {
1099                 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
1100                 pr_alert("rcu-scale types:");
1101                 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
1102                         pr_cont(" %s", scale_ops[i]->name);
1103                 pr_cont("\n");
1104                 firsterr = -EINVAL;
1105                 cur_ops = NULL;
1106                 goto unwind;
1107         }
1108         if (cur_ops->init)
1109                 if (!cur_ops->init()) {
1110                         firsterr = -EUCLEAN;
1111                         goto unwind;
1112                 }
1113
1114         ref_scale_print_module_parms(cur_ops, "Start of test");
1115
1116         // Shutdown task
1117         if (shutdown) {
1118                 init_waitqueue_head(&shutdown_wq);
1119                 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
1120                                                   shutdown_task);
1121                 if (torture_init_error(firsterr))
1122                         goto unwind;
1123                 schedule_timeout_uninterruptible(1);
1124         }
1125
1126         // Reader tasks (default to ~75% of online CPUs).
1127         if (nreaders < 0)
1128                 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
1129         if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
1130                 loops = 1;
1131         if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
1132                 nreaders = 1;
1133         if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
1134                 nruns = 1;
1135         reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
1136                                GFP_KERNEL);
1137         if (!reader_tasks) {
1138                 SCALEOUT_ERRSTRING("out of memory");
1139                 firsterr = -ENOMEM;
1140                 goto unwind;
1141         }
1142
1143         VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
1144
1145         for (i = 0; i < nreaders; i++) {
1146                 init_waitqueue_head(&reader_tasks[i].wq);
1147                 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
1148                                                   reader_tasks[i].task);
1149                 if (torture_init_error(firsterr))
1150                         goto unwind;
1151         }
1152
1153         // Main Task
1154         init_waitqueue_head(&main_wq);
1155         firsterr = torture_create_kthread(main_func, NULL, main_task);
1156         if (torture_init_error(firsterr))
1157                 goto unwind;
1158
1159         torture_init_end();
1160         return 0;
1161
1162 unwind:
1163         torture_init_end();
1164         ref_scale_cleanup();
1165         if (shutdown) {
1166                 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
1167                 kernel_power_off();
1168         }
1169         return firsterr;
1170 }
1171
1172 module_init(ref_scale_init);
1173 module_exit(ref_scale_cleanup);