net/core/devlink.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * net/core/devlink.c - Network physical/parent device Netlink interface
   4  *
   5  * Heavily inspired by net/wireless/
   6  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   7  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   8  */
   9
  10 #include <linux/etherdevice.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/types.h>
  14 #include <linux/slab.h>
  15 #include <linux/gfp.h>
  16 #include <linux/device.h>
  17 #include <linux/list.h>
  18 #include <linux/netdevice.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/refcount.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/u64_stats_sync.h>
  23 #include <linux/timekeeping.h>
  24 #include <rdma/ib_verbs.h>
  25 #include <net/netlink.h>
  26 #include <net/genetlink.h>
  27 #include <net/rtnetlink.h>
  28 #include <net/net_namespace.h>
  29 #include <net/sock.h>
  30 #include <net/devlink.h>
  31 #define CREATE_TRACE_POINTS
  32 #include <trace/events/devlink.h>
  33
  34 #define DEVLINK_RELOAD_STATS_ARRAY_SIZE \
  35         (__DEVLINK_RELOAD_LIMIT_MAX * __DEVLINK_RELOAD_ACTION_MAX)
  36
  37 struct devlink_dev_stats {
  38         u32 reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  39         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
  40 };
  41
  42 struct devlink {
  43         u32 index;
  44         struct list_head port_list;
  45         struct list_head rate_list;
  46         struct list_head sb_list;
  47         struct list_head dpipe_table_list;
  48         struct list_head resource_list;
  49         struct list_head param_list;
  50         struct list_head region_list;
  51         struct list_head reporter_list;
  52         struct mutex reporters_lock; /* protects reporter_list */
  53         struct devlink_dpipe_headers *dpipe_headers;
  54         struct list_head trap_list;
  55         struct list_head trap_group_list;
  56         struct list_head trap_policer_list;
  57         const struct devlink_ops *ops;
  58         u64 features;
  59         struct xarray snapshot_ids;
  60         struct devlink_dev_stats stats;
  61         struct device *dev;
  62         possible_net_t _net;
  63         /* Serializes access to devlink instance specific objects such as
  64          * port, sb, dpipe, resource, params, region, traps and more.
  65          */
  66         struct mutex lock;
  67         u8 reload_failed:1;
  68         refcount_t refcount;
  69         struct completion comp;
  70         char priv[] __aligned(NETDEV_ALIGN);
  71 };
  72
  73 /**
  74  * struct devlink_resource - devlink resource
  75  * @name: name of the resource
  76  * @id: id, per devlink instance
  77  * @size: size of the resource
  78  * @size_new: updated size of the resource, reload is needed
  79  * @size_valid: valid in case the total size of the resource is valid
  80  *              including its children
  81  * @parent: parent resource
  82  * @size_params: size parameters
  83  * @list: parent list
  84  * @resource_list: list of child resources
  85  * @occ_get: occupancy getter callback
  86  * @occ_get_priv: occupancy getter callback priv
  87  */
  88 struct devlink_resource {
  89         const char *name;
  90         u64 id;
  91         u64 size;
  92         u64 size_new;
  93         bool size_valid;
  94         struct devlink_resource *parent;
  95         struct devlink_resource_size_params size_params;
  96         struct list_head list;
  97         struct list_head resource_list;
  98         devlink_resource_occ_get_t *occ_get;
  99         void *occ_get_priv;
 100 };
 101
 102 void *devlink_priv(struct devlink *devlink)
 103 {
 104         return &devlink->priv;
 105 }
 106 EXPORT_SYMBOL_GPL(devlink_priv);
 107
 108 struct devlink *priv_to_devlink(void *priv)
 109 {
 110         return container_of(priv, struct devlink, priv);
 111 }
 112 EXPORT_SYMBOL_GPL(priv_to_devlink);
 113
 114 struct device *devlink_to_dev(const struct devlink *devlink)
 115 {
 116         return devlink->dev;
 117 }
 118 EXPORT_SYMBOL_GPL(devlink_to_dev);
 119
 120 static struct devlink_dpipe_field devlink_dpipe_fields_ethernet[] = {
 121         {
 122                 .name = "destination mac",
 123                 .id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC,
 124                 .bitwidth = 48,
 125         },
 126 };
 127
 128 struct devlink_dpipe_header devlink_dpipe_header_ethernet = {
 129         .name = "ethernet",
 130         .id = DEVLINK_DPIPE_HEADER_ETHERNET,
 131         .fields = devlink_dpipe_fields_ethernet,
 132         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ethernet),
 133         .global = true,
 134 };
 135 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ethernet);
 136
 137 static struct devlink_dpipe_field devlink_dpipe_fields_ipv4[] = {
 138         {
 139                 .name = "destination ip",
 140                 .id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP,
 141                 .bitwidth = 32,
 142         },
 143 };
 144
 145 struct devlink_dpipe_header devlink_dpipe_header_ipv4 = {
 146         .name = "ipv4",
 147         .id = DEVLINK_DPIPE_HEADER_IPV4,
 148         .fields = devlink_dpipe_fields_ipv4,
 149         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv4),
 150         .global = true,
 151 };
 152 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv4);
 153
 154 static struct devlink_dpipe_field devlink_dpipe_fields_ipv6[] = {
 155         {
 156                 .name = "destination ip",
 157                 .id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP,
 158                 .bitwidth = 128,
 159         },
 160 };
 161
 162 struct devlink_dpipe_header devlink_dpipe_header_ipv6 = {
 163         .name = "ipv6",
 164         .id = DEVLINK_DPIPE_HEADER_IPV6,
 165         .fields = devlink_dpipe_fields_ipv6,
 166         .fields_count = ARRAY_SIZE(devlink_dpipe_fields_ipv6),
 167         .global = true,
 168 };
 169 EXPORT_SYMBOL_GPL(devlink_dpipe_header_ipv6);
 170
 171 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
 172 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
 173 EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
 174
 175 static const struct nla_policy devlink_function_nl_policy[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1] = {
 176         [DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR] = { .type = NLA_BINARY },
 177         [DEVLINK_PORT_FN_ATTR_STATE] =
 178                 NLA_POLICY_RANGE(NLA_U8, DEVLINK_PORT_FN_STATE_INACTIVE,
 179                                  DEVLINK_PORT_FN_STATE_ACTIVE),
 180 };
 181
 182 static DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
 183 #define DEVLINK_REGISTERED XA_MARK_1
 184
 185 /* devlink instances are open to the access from the user space after
 186  * devlink_register() call. Such logical barrier allows us to have certain
 187  * expectations related to locking.
 188  *
 189  * Before *_register() - we are in initialization stage and no parallel
 190  * access possible to the devlink instance. All drivers perform that phase
 191  * by implicitly holding device_lock.
 192  *
 193  * After *_register() - users and driver can access devlink instance at
 194  * the same time.
 195  */
 196 #define ASSERT_DEVLINK_REGISTERED(d)                                           \
 197         WARN_ON_ONCE(!xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 198 #define ASSERT_DEVLINK_NOT_REGISTERED(d)                                       \
 199         WARN_ON_ONCE(xa_get_mark(&devlinks, (d)->index, DEVLINK_REGISTERED))
 200
 201 /* devlink_mutex
 202  *
 203  * An overall lock guarding every operation coming from userspace.
 204  * It also guards devlink devices list and it is taken when
 205  * driver registers/unregisters it.
 206  */
 207 static DEFINE_MUTEX(devlink_mutex);
 208
 209 struct net *devlink_net(const struct devlink *devlink)
 210 {
 211         return read_pnet(&devlink->_net);
 212 }
 213 EXPORT_SYMBOL_GPL(devlink_net);
 214
 215 void devlink_put(struct devlink *devlink)
 216 {
 217         if (refcount_dec_and_test(&devlink->refcount))
 218                 complete(&devlink->comp);
 219 }
 220
 221 struct devlink *__must_check devlink_try_get(struct devlink *devlink)
 222 {
 223         if (refcount_inc_not_zero(&devlink->refcount))
 224                 return devlink;
 225         return NULL;
 226 }
 227
 228 static struct devlink *devlink_get_from_attrs(struct net *net,
 229                                               struct nlattr **attrs)
 230 {
 231         struct devlink *devlink;
 232         unsigned long index;
 233         bool found = false;
 234         char *busname;
 235         char *devname;
 236
 237         if (!attrs[DEVLINK_ATTR_BUS_NAME] || !attrs[DEVLINK_ATTR_DEV_NAME])
 238                 return ERR_PTR(-EINVAL);
 239
 240         busname = nla_data(attrs[DEVLINK_ATTR_BUS_NAME]);
 241         devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 242
 243         lockdep_assert_held(&devlink_mutex);
 244
 245         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
 246                 if (strcmp(devlink->dev->bus->name, busname) == 0 &&
 247                     strcmp(dev_name(devlink->dev), devname) == 0 &&
 248                     net_eq(devlink_net(devlink), net)) {
 249                         found = true;
 250                         break;
 251                 }
 252         }
 253
 254         if (!found || !devlink_try_get(devlink))
 255                 devlink = ERR_PTR(-ENODEV);
 256
 257         return devlink;
 258 }
 259
 260 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 261                                                       unsigned int port_index)
 262 {
 263         struct devlink_port *devlink_port;
 264
 265         list_for_each_entry(devlink_port, &devlink->port_list, list) {
 266                 if (devlink_port->index == port_index)
 267                         return devlink_port;
 268         }
 269         return NULL;
 270 }
 271
 272 static bool devlink_port_index_exists(struct devlink *devlink,
 273                                       unsigned int port_index)
 274 {
 275         return devlink_port_get_by_index(devlink, port_index);
 276 }
 277
 278 static struct devlink_port *devlink_port_get_from_attrs(struct devlink *devlink,
 279                                                         struct nlattr **attrs)
 280 {
 281         if (attrs[DEVLINK_ATTR_PORT_INDEX]) {
 282                 u32 port_index = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
 283                 struct devlink_port *devlink_port;
 284
 285                 devlink_port = devlink_port_get_by_index(devlink, port_index);
 286                 if (!devlink_port)
 287                         return ERR_PTR(-ENODEV);
 288                 return devlink_port;
 289         }
 290         return ERR_PTR(-EINVAL);
 291 }
 292
 293 static struct devlink_port *devlink_port_get_from_info(struct devlink *devlink,
 294                                                        struct genl_info *info)
 295 {
 296         return devlink_port_get_from_attrs(devlink, info->attrs);
 297 }
 298
 299 static inline bool
 300 devlink_rate_is_leaf(struct devlink_rate *devlink_rate)
 301 {
 302         return devlink_rate->type == DEVLINK_RATE_TYPE_LEAF;
 303 }
 304
 305 static inline bool
 306 devlink_rate_is_node(struct devlink_rate *devlink_rate)
 307 {
 308         return devlink_rate->type == DEVLINK_RATE_TYPE_NODE;
 309 }
 310
 311 static struct devlink_rate *
 312 devlink_rate_leaf_get_from_info(struct devlink *devlink, struct genl_info *info)
 313 {
 314         struct devlink_rate *devlink_rate;
 315         struct devlink_port *devlink_port;
 316
 317         devlink_port = devlink_port_get_from_attrs(devlink, info->attrs);
 318         if (IS_ERR(devlink_port))
 319                 return ERR_CAST(devlink_port);
 320         devlink_rate = devlink_port->devlink_rate;
 321         return devlink_rate ?: ERR_PTR(-ENODEV);
 322 }
 323
 324 static struct devlink_rate *
 325 devlink_rate_node_get_by_name(struct devlink *devlink, const char *node_name)
 326 {
 327         static struct devlink_rate *devlink_rate;
 328
 329         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
 330                 if (devlink_rate_is_node(devlink_rate) &&
 331                     !strcmp(node_name, devlink_rate->name))
 332                         return devlink_rate;
 333         }
 334         return ERR_PTR(-ENODEV);
 335 }
 336
 337 static struct devlink_rate *
 338 devlink_rate_node_get_from_attrs(struct devlink *devlink, struct nlattr **attrs)
 339 {
 340         const char *rate_node_name;
 341         size_t len;
 342
 343         if (!attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 344                 return ERR_PTR(-EINVAL);
 345         rate_node_name = nla_data(attrs[DEVLINK_ATTR_RATE_NODE_NAME]);
 346         len = strlen(rate_node_name);
 347         /* Name cannot be empty or decimal number */
 348         if (!len || strspn(rate_node_name, "0123456789") == len)
 349                 return ERR_PTR(-EINVAL);
 350
 351         return devlink_rate_node_get_by_name(devlink, rate_node_name);
 352 }
 353
 354 static struct devlink_rate *
 355 devlink_rate_node_get_from_info(struct devlink *devlink, struct genl_info *info)
 356 {
 357         return devlink_rate_node_get_from_attrs(devlink, info->attrs);
 358 }
 359
 360 static struct devlink_rate *
 361 devlink_rate_get_from_info(struct devlink *devlink, struct genl_info *info)
 362 {
 363         struct nlattr **attrs = info->attrs;
 364
 365         if (attrs[DEVLINK_ATTR_PORT_INDEX])
 366                 return devlink_rate_leaf_get_from_info(devlink, info);
 367         else if (attrs[DEVLINK_ATTR_RATE_NODE_NAME])
 368                 return devlink_rate_node_get_from_info(devlink, info);
 369         else
 370                 return ERR_PTR(-EINVAL);
 371 }
 372
 373 struct devlink_sb {
 374         struct list_head list;
 375         unsigned int index;
 376         u32 size;
 377         u16 ingress_pools_count;
 378         u16 egress_pools_count;
 379         u16 ingress_tc_count;
 380         u16 egress_tc_count;
 381 };
 382
 383 static u16 devlink_sb_pool_count(struct devlink_sb *devlink_sb)
 384 {
 385         return devlink_sb->ingress_pools_count + devlink_sb->egress_pools_count;
 386 }
 387
 388 static struct devlink_sb *devlink_sb_get_by_index(struct devlink *devlink,
 389                                                   unsigned int sb_index)
 390 {
 391         struct devlink_sb *devlink_sb;
 392
 393         list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
 394                 if (devlink_sb->index == sb_index)
 395                         return devlink_sb;
 396         }
 397         return NULL;
 398 }
 399
 400 static bool devlink_sb_index_exists(struct devlink *devlink,
 401                                     unsigned int sb_index)
 402 {
 403         return devlink_sb_get_by_index(devlink, sb_index);
 404 }
 405
 406 static struct devlink_sb *devlink_sb_get_from_attrs(struct devlink *devlink,
 407                                                     struct nlattr **attrs)
 408 {
 409         if (attrs[DEVLINK_ATTR_SB_INDEX]) {
 410                 u32 sb_index = nla_get_u32(attrs[DEVLINK_ATTR_SB_INDEX]);
 411                 struct devlink_sb *devlink_sb;
 412
 413                 devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
 414                 if (!devlink_sb)
 415                         return ERR_PTR(-ENODEV);
 416                 return devlink_sb;
 417         }
 418         return ERR_PTR(-EINVAL);
 419 }
 420
 421 static struct devlink_sb *devlink_sb_get_from_info(struct devlink *devlink,
 422                                                    struct genl_info *info)
 423 {
 424         return devlink_sb_get_from_attrs(devlink, info->attrs);
 425 }
 426
 427 static int devlink_sb_pool_index_get_from_attrs(struct devlink_sb *devlink_sb,
 428                                                 struct nlattr **attrs,
 429                                                 u16 *p_pool_index)
 430 {
 431         u16 val;
 432
 433         if (!attrs[DEVLINK_ATTR_SB_POOL_INDEX])
 434                 return -EINVAL;
 435
 436         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_POOL_INDEX]);
 437         if (val >= devlink_sb_pool_count(devlink_sb))
 438                 return -EINVAL;
 439         *p_pool_index = val;
 440         return 0;
 441 }
 442
 443 static int devlink_sb_pool_index_get_from_info(struct devlink_sb *devlink_sb,
 444                                                struct genl_info *info,
 445                                                u16 *p_pool_index)
 446 {
 447         return devlink_sb_pool_index_get_from_attrs(devlink_sb, info->attrs,
 448                                                     p_pool_index);
 449 }
 450
 451 static int
 452 devlink_sb_pool_type_get_from_attrs(struct nlattr **attrs,
 453                                     enum devlink_sb_pool_type *p_pool_type)
 454 {
 455         u8 val;
 456
 457         if (!attrs[DEVLINK_ATTR_SB_POOL_TYPE])
 458                 return -EINVAL;
 459
 460         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_TYPE]);
 461         if (val != DEVLINK_SB_POOL_TYPE_INGRESS &&
 462             val != DEVLINK_SB_POOL_TYPE_EGRESS)
 463                 return -EINVAL;
 464         *p_pool_type = val;
 465         return 0;
 466 }
 467
 468 static int
 469 devlink_sb_pool_type_get_from_info(struct genl_info *info,
 470                                    enum devlink_sb_pool_type *p_pool_type)
 471 {
 472         return devlink_sb_pool_type_get_from_attrs(info->attrs, p_pool_type);
 473 }
 474
 475 static int
 476 devlink_sb_th_type_get_from_attrs(struct nlattr **attrs,
 477                                   enum devlink_sb_threshold_type *p_th_type)
 478 {
 479         u8 val;
 480
 481         if (!attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE])
 482                 return -EINVAL;
 483
 484         val = nla_get_u8(attrs[DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE]);
 485         if (val != DEVLINK_SB_THRESHOLD_TYPE_STATIC &&
 486             val != DEVLINK_SB_THRESHOLD_TYPE_DYNAMIC)
 487                 return -EINVAL;
 488         *p_th_type = val;
 489         return 0;
 490 }
 491
 492 static int
 493 devlink_sb_th_type_get_from_info(struct genl_info *info,
 494                                  enum devlink_sb_threshold_type *p_th_type)
 495 {
 496         return devlink_sb_th_type_get_from_attrs(info->attrs, p_th_type);
 497 }
 498
 499 static int
 500 devlink_sb_tc_index_get_from_attrs(struct devlink_sb *devlink_sb,
 501                                    struct nlattr **attrs,
 502                                    enum devlink_sb_pool_type pool_type,
 503                                    u16 *p_tc_index)
 504 {
 505         u16 val;
 506
 507         if (!attrs[DEVLINK_ATTR_SB_TC_INDEX])
 508                 return -EINVAL;
 509
 510         val = nla_get_u16(attrs[DEVLINK_ATTR_SB_TC_INDEX]);
 511         if (pool_type == DEVLINK_SB_POOL_TYPE_INGRESS &&
 512             val >= devlink_sb->ingress_tc_count)
 513                 return -EINVAL;
 514         if (pool_type == DEVLINK_SB_POOL_TYPE_EGRESS &&
 515             val >= devlink_sb->egress_tc_count)
 516                 return -EINVAL;
 517         *p_tc_index = val;
 518         return 0;
 519 }
 520
 521 static int
 522 devlink_sb_tc_index_get_from_info(struct devlink_sb *devlink_sb,
 523                                   struct genl_info *info,
 524                                   enum devlink_sb_pool_type pool_type,
 525                                   u16 *p_tc_index)
 526 {
 527         return devlink_sb_tc_index_get_from_attrs(devlink_sb, info->attrs,
 528                                                   pool_type, p_tc_index);
 529 }
 530
 531 struct devlink_region {
 532         struct devlink *devlink;
 533         struct devlink_port *port;
 534         struct list_head list;
 535         union {
 536                 const struct devlink_region_ops *ops;
 537                 const struct devlink_port_region_ops *port_ops;
 538         };
 539         struct list_head snapshot_list;
 540         u32 max_snapshots;
 541         u32 cur_snapshots;
 542         u64 size;
 543 };
 544
 545 struct devlink_snapshot {
 546         struct list_head list;
 547         struct devlink_region *region;
 548         u8 *data;
 549         u32 id;
 550 };
 551
 552 static struct devlink_region *
 553 devlink_region_get_by_name(struct devlink *devlink, const char *region_name)
 554 {
 555         struct devlink_region *region;
 556
 557         list_for_each_entry(region, &devlink->region_list, list)
 558                 if (!strcmp(region->ops->name, region_name))
 559                         return region;
 560
 561         return NULL;
 562 }
 563
 564 static struct devlink_region *
 565 devlink_port_region_get_by_name(struct devlink_port *port,
 566                                 const char *region_name)
 567 {
 568         struct devlink_region *region;
 569
 570         list_for_each_entry(region, &port->region_list, list)
 571                 if (!strcmp(region->ops->name, region_name))
 572                         return region;
 573
 574         return NULL;
 575 }
 576
 577 static struct devlink_snapshot *
 578 devlink_region_snapshot_get_by_id(struct devlink_region *region, u32 id)
 579 {
 580         struct devlink_snapshot *snapshot;
 581
 582         list_for_each_entry(snapshot, &region->snapshot_list, list)
 583                 if (snapshot->id == id)
 584                         return snapshot;
 585
 586         return NULL;
 587 }
 588
 589 #define DEVLINK_NL_FLAG_NEED_PORT               BIT(0)
 590 #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT    BIT(1)
 591 #define DEVLINK_NL_FLAG_NEED_RATE               BIT(2)
 592 #define DEVLINK_NL_FLAG_NEED_RATE_NODE          BIT(3)
 593
 594 /* The per devlink instance lock is taken by default in the pre-doit
 595  * operation, yet several commands do not require this. The global
 596  * devlink lock is taken and protects from disruption by user-calls.
 597  */
 598 #define DEVLINK_NL_FLAG_NO_LOCK                 BIT(4)
 599
 600 static int devlink_nl_pre_doit(const struct genl_ops *ops,
 601                                struct sk_buff *skb, struct genl_info *info)
 602 {
 603         struct devlink_port *devlink_port;
 604         struct devlink *devlink;
 605         int err;
 606
 607         mutex_lock(&devlink_mutex);
 608         devlink = devlink_get_from_attrs(genl_info_net(info), info->attrs);
 609         if (IS_ERR(devlink)) {
 610                 mutex_unlock(&devlink_mutex);
 611                 return PTR_ERR(devlink);
 612         }
 613         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 614                 mutex_lock(&devlink->lock);
 615         info->user_ptr[0] = devlink;
 616         if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_PORT) {
 617                 devlink_port = devlink_port_get_from_info(devlink, info);
 618                 if (IS_ERR(devlink_port)) {
 619                         err = PTR_ERR(devlink_port);
 620                         goto unlock;
 621                 }
 622                 info->user_ptr[1] = devlink_port;
 623         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT) {
 624                 devlink_port = devlink_port_get_from_info(devlink, info);
 625                 if (!IS_ERR(devlink_port))
 626                         info->user_ptr[1] = devlink_port;
 627         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE) {
 628                 struct devlink_rate *devlink_rate;
 629
 630                 devlink_rate = devlink_rate_get_from_info(devlink, info);
 631                 if (IS_ERR(devlink_rate)) {
 632                         err = PTR_ERR(devlink_rate);
 633                         goto unlock;
 634                 }
 635                 info->user_ptr[1] = devlink_rate;
 636         } else if (ops->internal_flags & DEVLINK_NL_FLAG_NEED_RATE_NODE) {
 637                 struct devlink_rate *rate_node;
 638
 639                 rate_node = devlink_rate_node_get_from_info(devlink, info);
 640                 if (IS_ERR(rate_node)) {
 641                         err = PTR_ERR(rate_node);
 642                         goto unlock;
 643                 }
 644                 info->user_ptr[1] = rate_node;
 645         }
 646         return 0;
 647
 648 unlock:
 649         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 650                 mutex_unlock(&devlink->lock);
 651         devlink_put(devlink);
 652         mutex_unlock(&devlink_mutex);
 653         return err;
 654 }
 655
 656 static void devlink_nl_post_doit(const struct genl_ops *ops,
 657                                  struct sk_buff *skb, struct genl_info *info)
 658 {
 659         struct devlink *devlink;
 660
 661         devlink = info->user_ptr[0];
 662         if (~ops->internal_flags & DEVLINK_NL_FLAG_NO_LOCK)
 663                 mutex_unlock(&devlink->lock);
 664         devlink_put(devlink);
 665         mutex_unlock(&devlink_mutex);
 666 }
 667
 668 static struct genl_family devlink_nl_family;
 669
 670 enum devlink_multicast_groups {
 671         DEVLINK_MCGRP_CONFIG,
 672 };
 673
 674 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 675         [DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 676 };
 677
 678 static int devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink)
 679 {
 680         if (nla_put_string(msg, DEVLINK_ATTR_BUS_NAME, devlink->dev->bus->name))
 681                 return -EMSGSIZE;
 682         if (nla_put_string(msg, DEVLINK_ATTR_DEV_NAME, dev_name(devlink->dev)))
 683                 return -EMSGSIZE;
 684         return 0;
 685 }
 686
 687 struct devlink_reload_combination {
 688         enum devlink_reload_action action;
 689         enum devlink_reload_limit limit;
 690 };
 691
 692 static const struct devlink_reload_combination devlink_reload_invalid_combinations[] = {
 693         {
 694                 /* can't reinitialize driver with no down time */
 695                 .action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 696                 .limit = DEVLINK_RELOAD_LIMIT_NO_RESET,
 697         },
 698 };
 699
 700 static bool
 701 devlink_reload_combination_is_invalid(enum devlink_reload_action action,
 702                                       enum devlink_reload_limit limit)
 703 {
 704         int i;
 705
 706         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)
 707                 if (devlink_reload_invalid_combinations[i].action == action &&
 708                     devlink_reload_invalid_combinations[i].limit == limit)
 709                         return true;
 710         return false;
 711 }
 712
 713 static bool
 714 devlink_reload_action_is_supported(struct devlink *devlink, enum devlink_reload_action action)
 715 {
 716         return test_bit(action, &devlink->ops->reload_actions);
 717 }
 718
 719 static bool
 720 devlink_reload_limit_is_supported(struct devlink *devlink, enum devlink_reload_limit limit)
 721 {
 722         return test_bit(limit, &devlink->ops->reload_limits);
 723 }
 724
 725 static int devlink_reload_stat_put(struct sk_buff *msg,
 726                                    enum devlink_reload_limit limit, u32 value)
 727 {
 728         struct nlattr *reload_stats_entry;
 729
 730         reload_stats_entry = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS_ENTRY);
 731         if (!reload_stats_entry)
 732                 return -EMSGSIZE;
 733
 734         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_STATS_LIMIT, limit) ||
 735             nla_put_u32(msg, DEVLINK_ATTR_RELOAD_STATS_VALUE, value))
 736                 goto nla_put_failure;
 737         nla_nest_end(msg, reload_stats_entry);
 738         return 0;
 739
 740 nla_put_failure:
 741         nla_nest_cancel(msg, reload_stats_entry);
 742         return -EMSGSIZE;
 743 }
 744
 745 static int devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_remote)
 746 {
 747         struct nlattr *reload_stats_attr, *act_info, *act_stats;
 748         int i, j, stat_idx;
 749         u32 value;
 750
 751         if (!is_remote)
 752                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_STATS);
 753         else
 754                 reload_stats_attr = nla_nest_start(msg, DEVLINK_ATTR_REMOTE_RELOAD_STATS);
 755
 756         if (!reload_stats_attr)
 757                 return -EMSGSIZE;
 758
 759         for (i = 0; i <= DEVLINK_RELOAD_ACTION_MAX; i++) {
 760                 if ((!is_remote &&
 761                      !devlink_reload_action_is_supported(devlink, i)) ||
 762                     i == DEVLINK_RELOAD_ACTION_UNSPEC)
 763                         continue;
 764                 act_info = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_INFO);
 765                 if (!act_info)
 766                         goto nla_put_failure;
 767
 768                 if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_ACTION, i))
 769                         goto action_info_nest_cancel;
 770                 act_stats = nla_nest_start(msg, DEVLINK_ATTR_RELOAD_ACTION_STATS);
 771                 if (!act_stats)
 772                         goto action_info_nest_cancel;
 773
 774                 for (j = 0; j <= DEVLINK_RELOAD_LIMIT_MAX; j++) {
 775                         /* Remote stats are shown even if not locally supported.
 776                          * Stats of actions with unspecified limit are shown
 777                          * though drivers don't need to register unspecified
 778                          * limit.
 779                          */
 780                         if ((!is_remote && j != DEVLINK_RELOAD_LIMIT_UNSPEC &&
 781                              !devlink_reload_limit_is_supported(devlink, j)) ||
 782                             devlink_reload_combination_is_invalid(i, j))
 783                                 continue;
 784
 785                         stat_idx = j * __DEVLINK_RELOAD_ACTION_MAX + i;
 786                         if (!is_remote)
 787                                 value = devlink->stats.reload_stats[stat_idx];
 788                         else
 789                                 value = devlink->stats.remote_reload_stats[stat_idx];
 790                         if (devlink_reload_stat_put(msg, j, value))
 791                                 goto action_stats_nest_cancel;
 792                 }
 793                 nla_nest_end(msg, act_stats);
 794                 nla_nest_end(msg, act_info);
 795         }
 796         nla_nest_end(msg, reload_stats_attr);
 797         return 0;
 798
 799 action_stats_nest_cancel:
 800         nla_nest_cancel(msg, act_stats);
 801 action_info_nest_cancel:
 802         nla_nest_cancel(msg, act_info);
 803 nla_put_failure:
 804         nla_nest_cancel(msg, reload_stats_attr);
 805         return -EMSGSIZE;
 806 }
 807
 808 static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink,
 809                            enum devlink_command cmd, u32 portid,
 810                            u32 seq, int flags)
 811 {
 812         struct nlattr *dev_stats;
 813         void *hdr;
 814
 815         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 816         if (!hdr)
 817                 return -EMSGSIZE;
 818
 819         if (devlink_nl_put_handle(msg, devlink))
 820                 goto nla_put_failure;
 821         if (nla_put_u8(msg, DEVLINK_ATTR_RELOAD_FAILED, devlink->reload_failed))
 822                 goto nla_put_failure;
 823
 824         dev_stats = nla_nest_start(msg, DEVLINK_ATTR_DEV_STATS);
 825         if (!dev_stats)
 826                 goto nla_put_failure;
 827
 828         if (devlink_reload_stats_put(msg, devlink, false))
 829                 goto dev_stats_nest_cancel;
 830         if (devlink_reload_stats_put(msg, devlink, true))
 831                 goto dev_stats_nest_cancel;
 832
 833         nla_nest_end(msg, dev_stats);
 834         genlmsg_end(msg, hdr);
 835         return 0;
 836
 837 dev_stats_nest_cancel:
 838         nla_nest_cancel(msg, dev_stats);
 839 nla_put_failure:
 840         genlmsg_cancel(msg, hdr);
 841         return -EMSGSIZE;
 842 }
 843
 844 static void devlink_notify(struct devlink *devlink, enum devlink_command cmd)
 845 {
 846         struct sk_buff *msg;
 847         int err;
 848
 849         WARN_ON(cmd != DEVLINK_CMD_NEW && cmd != DEVLINK_CMD_DEL);
 850         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
 851
 852         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 853         if (!msg)
 854                 return;
 855
 856         err = devlink_nl_fill(msg, devlink, cmd, 0, 0, 0);
 857         if (err) {
 858                 nlmsg_free(msg);
 859                 return;
 860         }
 861
 862         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
 863                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
 864 }
 865
 866 static int devlink_nl_port_attrs_put(struct sk_buff *msg,
 867                                      struct devlink_port *devlink_port)
 868 {
 869         struct devlink_port_attrs *attrs = &devlink_port->attrs;
 870
 871         if (!devlink_port->attrs_set)
 872                 return 0;
 873         if (attrs->lanes) {
 874                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_LANES, attrs->lanes))
 875                         return -EMSGSIZE;
 876         }
 877         if (nla_put_u8(msg, DEVLINK_ATTR_PORT_SPLITTABLE, attrs->splittable))
 878                 return -EMSGSIZE;
 879         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_FLAVOUR, attrs->flavour))
 880                 return -EMSGSIZE;
 881         switch (devlink_port->attrs.flavour) {
 882         case DEVLINK_PORT_FLAVOUR_PCI_PF:
 883                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 884                                 attrs->pci_pf.controller) ||
 885                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
 886                         return -EMSGSIZE;
 887                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
 888                         return -EMSGSIZE;
 889                 break;
 890         case DEVLINK_PORT_FLAVOUR_PCI_VF:
 891                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 892                                 attrs->pci_vf.controller) ||
 893                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
 894                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
 895                         return -EMSGSIZE;
 896                 if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
 897                         return -EMSGSIZE;
 898                 break;
 899         case DEVLINK_PORT_FLAVOUR_PCI_SF:
 900                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
 901                                 attrs->pci_sf.controller) ||
 902                     nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
 903                                 attrs->pci_sf.pf) ||
 904                     nla_put_u32(msg, DEVLINK_ATTR_PORT_PCI_SF_NUMBER,
 905                                 attrs->pci_sf.sf))
 906                         return -EMSGSIZE;
 907                 break;
 908         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
 909         case DEVLINK_PORT_FLAVOUR_CPU:
 910         case DEVLINK_PORT_FLAVOUR_DSA:
 911                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_NUMBER,
 912                                 attrs->phys.port_number))
 913                         return -EMSGSIZE;
 914                 if (!attrs->split)
 915                         return 0;
 916                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_GROUP,
 917                                 attrs->phys.port_number))
 918                         return -EMSGSIZE;
 919                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_SPLIT_SUBPORT_NUMBER,
 920                                 attrs->phys.split_subport_number))
 921                         return -EMSGSIZE;
 922                 break;
 923         default:
 924                 break;
 925         }
 926         return 0;
 927 }
 928
 929 static int devlink_port_fn_hw_addr_fill(const struct devlink_ops *ops,
 930                                         struct devlink_port *port,
 931                                         struct sk_buff *msg,
 932                                         struct netlink_ext_ack *extack,
 933                                         bool *msg_updated)
 934 {
 935         u8 hw_addr[MAX_ADDR_LEN];
 936         int hw_addr_len;
 937         int err;
 938
 939         if (!ops->port_function_hw_addr_get)
 940                 return 0;
 941
 942         err = ops->port_function_hw_addr_get(port, hw_addr, &hw_addr_len,
 943                                              extack);
 944         if (err) {
 945                 if (err == -EOPNOTSUPP)
 946                         return 0;
 947                 return err;
 948         }
 949         err = nla_put(msg, DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR, hw_addr_len, hw_addr);
 950         if (err)
 951                 return err;
 952         *msg_updated = true;
 953         return 0;
 954 }
 955
 956 static int devlink_nl_rate_fill(struct sk_buff *msg,
 957                                 struct devlink_rate *devlink_rate,
 958                                 enum devlink_command cmd, u32 portid, u32 seq,
 959                                 int flags, struct netlink_ext_ack *extack)
 960 {
 961         struct devlink *devlink = devlink_rate->devlink;
 962         void *hdr;
 963
 964         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
 965         if (!hdr)
 966                 return -EMSGSIZE;
 967
 968         if (devlink_nl_put_handle(msg, devlink))
 969                 goto nla_put_failure;
 970
 971         if (nla_put_u16(msg, DEVLINK_ATTR_RATE_TYPE, devlink_rate->type))
 972                 goto nla_put_failure;
 973
 974         if (devlink_rate_is_leaf(devlink_rate)) {
 975                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
 976                                 devlink_rate->devlink_port->index))
 977                         goto nla_put_failure;
 978         } else if (devlink_rate_is_node(devlink_rate)) {
 979                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_NODE_NAME,
 980                                    devlink_rate->name))
 981                         goto nla_put_failure;
 982         }
 983
 984         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_SHARE,
 985                               devlink_rate->tx_share, DEVLINK_ATTR_PAD))
 986                 goto nla_put_failure;
 987
 988         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_RATE_TX_MAX,
 989                               devlink_rate->tx_max, DEVLINK_ATTR_PAD))
 990                 goto nla_put_failure;
 991
 992         if (devlink_rate->parent)
 993                 if (nla_put_string(msg, DEVLINK_ATTR_RATE_PARENT_NODE_NAME,
 994                                    devlink_rate->parent->name))
 995                         goto nla_put_failure;
 996
 997         genlmsg_end(msg, hdr);
 998         return 0;
 999
1000 nla_put_failure:
1001         genlmsg_cancel(msg, hdr);
1002         return -EMSGSIZE;
1003 }
1004
1005 static bool
1006 devlink_port_fn_state_valid(enum devlink_port_fn_state state)
1007 {
1008         return state == DEVLINK_PORT_FN_STATE_INACTIVE ||
1009                state == DEVLINK_PORT_FN_STATE_ACTIVE;
1010 }
1011
1012 static bool
1013 devlink_port_fn_opstate_valid(enum devlink_port_fn_opstate opstate)
1014 {
1015         return opstate == DEVLINK_PORT_FN_OPSTATE_DETACHED ||
1016                opstate == DEVLINK_PORT_FN_OPSTATE_ATTACHED;
1017 }
1018
1019 static int devlink_port_fn_state_fill(const struct devlink_ops *ops,
1020                                       struct devlink_port *port,
1021                                       struct sk_buff *msg,
1022                                       struct netlink_ext_ack *extack,
1023                                       bool *msg_updated)
1024 {
1025         enum devlink_port_fn_opstate opstate;
1026         enum devlink_port_fn_state state;
1027         int err;
1028
1029         if (!ops->port_fn_state_get)
1030                 return 0;
1031
1032         err = ops->port_fn_state_get(port, &state, &opstate, extack);
1033         if (err) {
1034                 if (err == -EOPNOTSUPP)
1035                         return 0;
1036                 return err;
1037         }
1038         if (!devlink_port_fn_state_valid(state)) {
1039                 WARN_ON_ONCE(1);
1040                 NL_SET_ERR_MSG_MOD(extack, "Invalid state read from driver");
1041                 return -EINVAL;
1042         }
1043         if (!devlink_port_fn_opstate_valid(opstate)) {
1044                 WARN_ON_ONCE(1);
1045                 NL_SET_ERR_MSG_MOD(extack,
1046                                    "Invalid operational state read from driver");
1047                 return -EINVAL;
1048         }
1049         if (nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_STATE, state) ||
1050             nla_put_u8(msg, DEVLINK_PORT_FN_ATTR_OPSTATE, opstate))
1051                 return -EMSGSIZE;
1052         *msg_updated = true;
1053         return 0;
1054 }
1055
1056 static int
1057 devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *port,
1058                                    struct netlink_ext_ack *extack)
1059 {
1060         const struct devlink_ops *ops;
1061         struct nlattr *function_attr;
1062         bool msg_updated = false;
1063         int err;
1064
1065         function_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PORT_FUNCTION);
1066         if (!function_attr)
1067                 return -EMSGSIZE;
1068
1069         ops = port->devlink->ops;
1070         err = devlink_port_fn_hw_addr_fill(ops, port, msg, extack,
1071                                            &msg_updated);
1072         if (err)
1073                 goto out;
1074         err = devlink_port_fn_state_fill(ops, port, msg, extack, &msg_updated);
1075 out:
1076         if (err || !msg_updated)
1077                 nla_nest_cancel(msg, function_attr);
1078         else
1079                 nla_nest_end(msg, function_attr);
1080         return err;
1081 }
1082
1083 static int devlink_nl_port_fill(struct sk_buff *msg,
1084                                 struct devlink_port *devlink_port,
1085                                 enum devlink_command cmd, u32 portid, u32 seq,
1086                                 int flags, struct netlink_ext_ack *extack)
1087 {
1088         struct devlink *devlink = devlink_port->devlink;
1089         void *hdr;
1090
1091         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1092         if (!hdr)
1093                 return -EMSGSIZE;
1094
1095         if (devlink_nl_put_handle(msg, devlink))
1096                 goto nla_put_failure;
1097         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
1098                 goto nla_put_failure;
1099
1100         /* Hold rtnl lock while accessing port's netdev attributes. */
1101         rtnl_lock();
1102         spin_lock_bh(&devlink_port->type_lock);
1103         if (nla_put_u16(msg, DEVLINK_ATTR_PORT_TYPE, devlink_port->type))
1104                 goto nla_put_failure_type_locked;
1105         if (devlink_port->desired_type != DEVLINK_PORT_TYPE_NOTSET &&
1106             nla_put_u16(msg, DEVLINK_ATTR_PORT_DESIRED_TYPE,
1107                         devlink_port->desired_type))
1108                 goto nla_put_failure_type_locked;
1109         if (devlink_port->type == DEVLINK_PORT_TYPE_ETH) {
1110                 struct net *net = devlink_net(devlink_port->devlink);
1111                 struct net_device *netdev = devlink_port->type_dev;
1112
1113                 if (netdev && net_eq(net, dev_net(netdev)) &&
1114                     (nla_put_u32(msg, DEVLINK_ATTR_PORT_NETDEV_IFINDEX,
1115                                  netdev->ifindex) ||
1116                      nla_put_string(msg, DEVLINK_ATTR_PORT_NETDEV_NAME,
1117                                     netdev->name)))
1118                         goto nla_put_failure_type_locked;
1119         }
1120         if (devlink_port->type == DEVLINK_PORT_TYPE_IB) {
1121                 struct ib_device *ibdev = devlink_port->type_dev;
1122
1123                 if (ibdev &&
1124                     nla_put_string(msg, DEVLINK_ATTR_PORT_IBDEV_NAME,
1125                                    ibdev->name))
1126                         goto nla_put_failure_type_locked;
1127         }
1128         spin_unlock_bh(&devlink_port->type_lock);
1129         rtnl_unlock();
1130         if (devlink_nl_port_attrs_put(msg, devlink_port))
1131                 goto nla_put_failure;
1132         if (devlink_nl_port_function_attrs_put(msg, devlink_port, extack))
1133                 goto nla_put_failure;
1134
1135         genlmsg_end(msg, hdr);
1136         return 0;
1137
1138 nla_put_failure_type_locked:
1139         spin_unlock_bh(&devlink_port->type_lock);
1140         rtnl_unlock();
1141 nla_put_failure:
1142         genlmsg_cancel(msg, hdr);
1143         return -EMSGSIZE;
1144 }
1145
1146 static void devlink_port_notify(struct devlink_port *devlink_port,
1147                                 enum devlink_command cmd)
1148 {
1149         struct devlink *devlink = devlink_port->devlink;
1150         struct sk_buff *msg;
1151         int err;
1152
1153         WARN_ON(cmd != DEVLINK_CMD_PORT_NEW && cmd != DEVLINK_CMD_PORT_DEL);
1154
1155         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1156                 return;
1157
1158         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1159         if (!msg)
1160                 return;
1161
1162         err = devlink_nl_port_fill(msg, devlink_port, cmd, 0, 0, 0, NULL);
1163         if (err) {
1164                 nlmsg_free(msg);
1165                 return;
1166         }
1167
1168         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1169                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1170 }
1171
1172 static void devlink_rate_notify(struct devlink_rate *devlink_rate,
1173                                 enum devlink_command cmd)
1174 {
1175         struct devlink *devlink = devlink_rate->devlink;
1176         struct sk_buff *msg;
1177         int err;
1178
1179         WARN_ON(cmd != DEVLINK_CMD_RATE_NEW && cmd != DEVLINK_CMD_RATE_DEL);
1180
1181         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
1182                 return;
1183
1184         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1185         if (!msg)
1186                 return;
1187
1188         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, 0, 0, 0, NULL);
1189         if (err) {
1190                 nlmsg_free(msg);
1191                 return;
1192         }
1193
1194         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
1195                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
1196 }
1197
1198 static int devlink_nl_cmd_rate_get_dumpit(struct sk_buff *msg,
1199                                           struct netlink_callback *cb)
1200 {
1201         struct devlink_rate *devlink_rate;
1202         struct devlink *devlink;
1203         int start = cb->args[0];
1204         unsigned long index;
1205         int idx = 0;
1206         int err = 0;
1207
1208         mutex_lock(&devlink_mutex);
1209         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1210                 if (!devlink_try_get(devlink))
1211                         continue;
1212
1213                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1214                         goto retry;
1215
1216                 mutex_lock(&devlink->lock);
1217                 list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
1218                         enum devlink_command cmd = DEVLINK_CMD_RATE_NEW;
1219                         u32 id = NETLINK_CB(cb->skb).portid;
1220
1221                         if (idx < start) {
1222                                 idx++;
1223                                 continue;
1224                         }
1225                         err = devlink_nl_rate_fill(msg, devlink_rate, cmd, id,
1226                                                    cb->nlh->nlmsg_seq,
1227                                                    NLM_F_MULTI, NULL);
1228                         if (err) {
1229                                 mutex_unlock(&devlink->lock);
1230                                 devlink_put(devlink);
1231                                 goto out;
1232                         }
1233                         idx++;
1234                 }
1235                 mutex_unlock(&devlink->lock);
1236 retry:
1237                 devlink_put(devlink);
1238         }
1239 out:
1240         mutex_unlock(&devlink_mutex);
1241         if (err != -EMSGSIZE)
1242                 return err;
1243
1244         cb->args[0] = idx;
1245         return msg->len;
1246 }
1247
1248 static int devlink_nl_cmd_rate_get_doit(struct sk_buff *skb,
1249                                         struct genl_info *info)
1250 {
1251         struct devlink_rate *devlink_rate = info->user_ptr[1];
1252         struct sk_buff *msg;
1253         int err;
1254
1255         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1256         if (!msg)
1257                 return -ENOMEM;
1258
1259         err = devlink_nl_rate_fill(msg, devlink_rate, DEVLINK_CMD_RATE_NEW,
1260                                    info->snd_portid, info->snd_seq, 0,
1261                                    info->extack);
1262         if (err) {
1263                 nlmsg_free(msg);
1264                 return err;
1265         }
1266
1267         return genlmsg_reply(msg, info);
1268 }
1269
1270 static bool
1271 devlink_rate_is_parent_node(struct devlink_rate *devlink_rate,
1272                             struct devlink_rate *parent)
1273 {
1274         while (parent) {
1275                 if (parent == devlink_rate)
1276                         return true;
1277                 parent = parent->parent;
1278         }
1279         return false;
1280 }
1281
1282 static int devlink_nl_cmd_get_doit(struct sk_buff *skb, struct genl_info *info)
1283 {
1284         struct devlink *devlink = info->user_ptr[0];
1285         struct sk_buff *msg;
1286         int err;
1287
1288         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1289         if (!msg)
1290                 return -ENOMEM;
1291
1292         err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1293                               info->snd_portid, info->snd_seq, 0);
1294         if (err) {
1295                 nlmsg_free(msg);
1296                 return err;
1297         }
1298
1299         return genlmsg_reply(msg, info);
1300 }
1301
1302 static int devlink_nl_cmd_get_dumpit(struct sk_buff *msg,
1303                                      struct netlink_callback *cb)
1304 {
1305         struct devlink *devlink;
1306         int start = cb->args[0];
1307         unsigned long index;
1308         int idx = 0;
1309         int err;
1310
1311         mutex_lock(&devlink_mutex);
1312         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1313                 if (!devlink_try_get(devlink))
1314                         continue;
1315
1316                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk))) {
1317                         devlink_put(devlink);
1318                         continue;
1319                 }
1320
1321                 if (idx < start) {
1322                         idx++;
1323                         devlink_put(devlink);
1324                         continue;
1325                 }
1326
1327                 err = devlink_nl_fill(msg, devlink, DEVLINK_CMD_NEW,
1328                                       NETLINK_CB(cb->skb).portid,
1329                                       cb->nlh->nlmsg_seq, NLM_F_MULTI);
1330                 devlink_put(devlink);
1331                 if (err)
1332                         goto out;
1333                 idx++;
1334         }
1335 out:
1336         mutex_unlock(&devlink_mutex);
1337
1338         cb->args[0] = idx;
1339         return msg->len;
1340 }
1341
1342 static int devlink_nl_cmd_port_get_doit(struct sk_buff *skb,
1343                                         struct genl_info *info)
1344 {
1345         struct devlink_port *devlink_port = info->user_ptr[1];
1346         struct sk_buff *msg;
1347         int err;
1348
1349         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1350         if (!msg)
1351                 return -ENOMEM;
1352
1353         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_PORT_NEW,
1354                                    info->snd_portid, info->snd_seq, 0,
1355                                    info->extack);
1356         if (err) {
1357                 nlmsg_free(msg);
1358                 return err;
1359         }
1360
1361         return genlmsg_reply(msg, info);
1362 }
1363
1364 static int devlink_nl_cmd_port_get_dumpit(struct sk_buff *msg,
1365                                           struct netlink_callback *cb)
1366 {
1367         struct devlink *devlink;
1368         struct devlink_port *devlink_port;
1369         int start = cb->args[0];
1370         unsigned long index;
1371         int idx = 0;
1372         int err;
1373
1374         mutex_lock(&devlink_mutex);
1375         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
1376                 if (!devlink_try_get(devlink))
1377                         continue;
1378
1379                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
1380                         goto retry;
1381
1382                 mutex_lock(&devlink->lock);
1383                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
1384                         if (idx < start) {
1385                                 idx++;
1386                                 continue;
1387                         }
1388                         err = devlink_nl_port_fill(msg, devlink_port,
1389                                                    DEVLINK_CMD_NEW,
1390                                                    NETLINK_CB(cb->skb).portid,
1391                                                    cb->nlh->nlmsg_seq,
1392                                                    NLM_F_MULTI, cb->extack);
1393                         if (err) {
1394                                 mutex_unlock(&devlink->lock);
1395                                 devlink_put(devlink);
1396                                 goto out;
1397                         }
1398                         idx++;
1399                 }
1400                 mutex_unlock(&devlink->lock);
1401 retry:
1402                 devlink_put(devlink);
1403         }
1404 out:
1405         mutex_unlock(&devlink_mutex);
1406
1407         cb->args[0] = idx;
1408         return msg->len;
1409 }
1410
1411 static int devlink_port_type_set(struct devlink_port *devlink_port,
1412                                  enum devlink_port_type port_type)
1413
1414 {
1415         int err;
1416
1417         if (!devlink_port->devlink->ops->port_type_set)
1418                 return -EOPNOTSUPP;
1419
1420         if (port_type == devlink_port->type)
1421                 return 0;
1422
1423         err = devlink_port->devlink->ops->port_type_set(devlink_port,
1424                                                         port_type);
1425         if (err)
1426                 return err;
1427
1428         devlink_port->desired_type = port_type;
1429         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
1430         return 0;
1431 }
1432
1433 static int devlink_port_function_hw_addr_set(struct devlink_port *port,
1434                                              const struct nlattr *attr,
1435                                              struct netlink_ext_ack *extack)
1436 {
1437         const struct devlink_ops *ops = port->devlink->ops;
1438         const u8 *hw_addr;
1439         int hw_addr_len;
1440
1441         hw_addr = nla_data(attr);
1442         hw_addr_len = nla_len(attr);
1443         if (hw_addr_len > MAX_ADDR_LEN) {
1444                 NL_SET_ERR_MSG_MOD(extack, "Port function hardware address too long");
1445                 return -EINVAL;
1446         }
1447         if (port->type == DEVLINK_PORT_TYPE_ETH) {
1448                 if (hw_addr_len != ETH_ALEN) {
1449                         NL_SET_ERR_MSG_MOD(extack, "Address must be 6 bytes for Ethernet device");
1450                         return -EINVAL;
1451                 }
1452                 if (!is_unicast_ether_addr(hw_addr)) {
1453                         NL_SET_ERR_MSG_MOD(extack, "Non-unicast hardware address unsupported");
1454                         return -EINVAL;
1455                 }
1456         }
1457
1458         if (!ops->port_function_hw_addr_set) {
1459                 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support function attributes");
1460                 return -EOPNOTSUPP;
1461         }
1462
1463         return ops->port_function_hw_addr_set(port, hw_addr, hw_addr_len,
1464                                               extack);
1465 }
1466
1467 static int devlink_port_fn_state_set(struct devlink_port *port,
1468                                      const struct nlattr *attr,
1469                                      struct netlink_ext_ack *extack)
1470 {
1471         enum devlink_port_fn_state state;
1472         const struct devlink_ops *ops;
1473
1474         state = nla_get_u8(attr);
1475         ops = port->devlink->ops;
1476         if (!ops->port_fn_state_set) {
1477                 NL_SET_ERR_MSG_MOD(extack,
1478                                    "Function does not support state setting");
1479                 return -EOPNOTSUPP;
1480         }
1481         return ops->port_fn_state_set(port, state, extack);
1482 }
1483
1484 static int devlink_port_function_set(struct devlink_port *port,
1485                                      const struct nlattr *attr,
1486                                      struct netlink_ext_ack *extack)
1487 {
1488         struct nlattr *tb[DEVLINK_PORT_FUNCTION_ATTR_MAX + 1];
1489         int err;
1490
1491         err = nla_parse_nested(tb, DEVLINK_PORT_FUNCTION_ATTR_MAX, attr,
1492                                devlink_function_nl_policy, extack);
1493         if (err < 0) {
1494                 NL_SET_ERR_MSG_MOD(extack, "Fail to parse port function attributes");
1495                 return err;
1496         }
1497
1498         attr = tb[DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR];
1499         if (attr) {
1500                 err = devlink_port_function_hw_addr_set(port, attr, extack);
1501                 if (err)
1502                         return err;
1503         }
1504         /* Keep this as the last function attribute set, so that when
1505          * multiple port function attributes are set along with state,
1506          * Those can be applied first before activating the state.
1507          */
1508         attr = tb[DEVLINK_PORT_FN_ATTR_STATE];
1509         if (attr)
1510                 err = devlink_port_fn_state_set(port, attr, extack);
1511
1512         if (!err)
1513                 devlink_port_notify(port, DEVLINK_CMD_PORT_NEW);
1514         return err;
1515 }
1516
1517 static int devlink_nl_cmd_port_set_doit(struct sk_buff *skb,
1518                                         struct genl_info *info)
1519 {
1520         struct devlink_port *devlink_port = info->user_ptr[1];
1521         int err;
1522
1523         if (info->attrs[DEVLINK_ATTR_PORT_TYPE]) {
1524                 enum devlink_port_type port_type;
1525
1526                 port_type = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_TYPE]);
1527                 err = devlink_port_type_set(devlink_port, port_type);
1528                 if (err)
1529                         return err;
1530         }
1531
1532         if (info->attrs[DEVLINK_ATTR_PORT_FUNCTION]) {
1533                 struct nlattr *attr = info->attrs[DEVLINK_ATTR_PORT_FUNCTION];
1534                 struct netlink_ext_ack *extack = info->extack;
1535
1536                 err = devlink_port_function_set(devlink_port, attr, extack);
1537                 if (err)
1538                         return err;
1539         }
1540
1541         return 0;
1542 }
1543
1544 static int devlink_port_split(struct devlink *devlink, u32 port_index,
1545                               u32 count, struct netlink_ext_ack *extack)
1546
1547 {
1548         if (devlink->ops->port_split)
1549                 return devlink->ops->port_split(devlink, port_index, count,
1550                                                 extack);
1551         return -EOPNOTSUPP;
1552 }
1553
1554 static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
1555                                           struct genl_info *info)
1556 {
1557         struct devlink *devlink = info->user_ptr[0];
1558         struct devlink_port *devlink_port;
1559         u32 port_index;
1560         u32 count;
1561
1562         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX] ||
1563             !info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
1564                 return -EINVAL;
1565
1566         devlink_port = devlink_port_get_from_info(devlink, info);
1567         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1568         count = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]);
1569
1570         if (IS_ERR(devlink_port))
1571                 return -EINVAL;
1572
1573         if (!devlink_port->attrs.splittable) {
1574                 /* Split ports cannot be split. */
1575                 if (devlink_port->attrs.split)
1576                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split further");
1577                 else
1578                         NL_SET_ERR_MSG_MOD(info->extack, "Port cannot be split");
1579                 return -EINVAL;
1580         }
1581
1582         if (count < 2 || !is_power_of_2(count) || count > devlink_port->attrs.lanes) {
1583                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid split count");
1584                 return -EINVAL;
1585         }
1586
1587         return devlink_port_split(devlink, port_index, count, info->extack);
1588 }
1589
1590 static int devlink_port_unsplit(struct devlink *devlink, u32 port_index,
1591                                 struct netlink_ext_ack *extack)
1592
1593 {
1594         if (devlink->ops->port_unsplit)
1595                 return devlink->ops->port_unsplit(devlink, port_index, extack);
1596         return -EOPNOTSUPP;
1597 }
1598
1599 static int devlink_nl_cmd_port_unsplit_doit(struct sk_buff *skb,
1600                                             struct genl_info *info)
1601 {
1602         struct devlink *devlink = info->user_ptr[0];
1603         u32 port_index;
1604
1605         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX])
1606                 return -EINVAL;
1607
1608         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1609         return devlink_port_unsplit(devlink, port_index, info->extack);
1610 }
1611
1612 static int devlink_port_new_notifiy(struct devlink *devlink,
1613                                     unsigned int port_index,
1614                                     struct genl_info *info)
1615 {
1616         struct devlink_port *devlink_port;
1617         struct sk_buff *msg;
1618         int err;
1619
1620         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1621         if (!msg)
1622                 return -ENOMEM;
1623
1624         mutex_lock(&devlink->lock);
1625         devlink_port = devlink_port_get_by_index(devlink, port_index);
1626         if (!devlink_port) {
1627                 err = -ENODEV;
1628                 goto out;
1629         }
1630
1631         err = devlink_nl_port_fill(msg, devlink_port, DEVLINK_CMD_NEW,
1632                                    info->snd_portid, info->snd_seq, 0, NULL);
1633         if (err)
1634                 goto out;
1635
1636         err = genlmsg_reply(msg, info);
1637         mutex_unlock(&devlink->lock);
1638         return err;
1639
1640 out:
1641         mutex_unlock(&devlink->lock);
1642         nlmsg_free(msg);
1643         return err;
1644 }
1645
1646 static int devlink_nl_cmd_port_new_doit(struct sk_buff *skb,
1647                                         struct genl_info *info)
1648 {
1649         struct netlink_ext_ack *extack = info->extack;
1650         struct devlink_port_new_attrs new_attrs = {};
1651         struct devlink *devlink = info->user_ptr[0];
1652         unsigned int new_port_index;
1653         int err;
1654
1655         if (!devlink->ops->port_new || !devlink->ops->port_del)
1656                 return -EOPNOTSUPP;
1657
1658         if (!info->attrs[DEVLINK_ATTR_PORT_FLAVOUR] ||
1659             !info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]) {
1660                 NL_SET_ERR_MSG_MOD(extack, "Port flavour or PCI PF are not specified");
1661                 return -EINVAL;
1662         }
1663         new_attrs.flavour = nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_FLAVOUR]);
1664         new_attrs.pfnum =
1665                 nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_PCI_PF_NUMBER]);
1666
1667         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1668                 /* Port index of the new port being created by driver. */
1669                 new_attrs.port_index =
1670                         nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1671                 new_attrs.port_index_valid = true;
1672         }
1673         if (info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]) {
1674                 new_attrs.controller =
1675                         nla_get_u16(info->attrs[DEVLINK_ATTR_PORT_CONTROLLER_NUMBER]);
1676                 new_attrs.controller_valid = true;
1677         }
1678         if (new_attrs.flavour == DEVLINK_PORT_FLAVOUR_PCI_SF &&
1679             info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]) {
1680                 new_attrs.sfnum = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_PCI_SF_NUMBER]);
1681                 new_attrs.sfnum_valid = true;
1682         }
1683
1684         err = devlink->ops->port_new(devlink, &new_attrs, extack,
1685                                      &new_port_index);
1686         if (err)
1687                 return err;
1688
1689         err = devlink_port_new_notifiy(devlink, new_port_index, info);
1690         if (err && err != -ENODEV) {
1691                 /* Fail to send the response; destroy newly created port. */
1692                 devlink->ops->port_del(devlink, new_port_index, extack);
1693         }
1694         return err;
1695 }
1696
1697 static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
1698                                         struct genl_info *info)
1699 {
1700         struct netlink_ext_ack *extack = info->extack;
1701         struct devlink *devlink = info->user_ptr[0];
1702         unsigned int port_index;
1703
1704         if (!devlink->ops->port_del)
1705                 return -EOPNOTSUPP;
1706
1707         if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
1708                 NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
1709                 return -EINVAL;
1710         }
1711         port_index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
1712
1713         return devlink->ops->port_del(devlink, port_index, extack);
1714 }
1715
1716 static int
1717 devlink_nl_rate_parent_node_set(struct devlink_rate *devlink_rate,
1718                                 struct genl_info *info,
1719                                 struct nlattr *nla_parent)
1720 {
1721         struct devlink *devlink = devlink_rate->devlink;
1722         const char *parent_name = nla_data(nla_parent);
1723         const struct devlink_ops *ops = devlink->ops;
1724         size_t len = strlen(parent_name);
1725         struct devlink_rate *parent;
1726         int err = -EOPNOTSUPP;
1727
1728         parent = devlink_rate->parent;
1729         if (parent && len) {
1730                 NL_SET_ERR_MSG_MOD(info->extack, "Rate object already has parent.");
1731                 return -EBUSY;
1732         } else if (parent && !len) {
1733                 if (devlink_rate_is_leaf(devlink_rate))
1734                         err = ops->rate_leaf_parent_set(devlink_rate, NULL,
1735                                                         devlink_rate->priv, NULL,
1736                                                         info->extack);
1737                 else if (devlink_rate_is_node(devlink_rate))
1738                         err = ops->rate_node_parent_set(devlink_rate, NULL,
1739                                                         devlink_rate->priv, NULL,
1740                                                         info->extack);
1741                 if (err)
1742                         return err;
1743
1744                 refcount_dec(&parent->refcnt);
1745                 devlink_rate->parent = NULL;
1746         } else if (!parent && len) {
1747                 parent = devlink_rate_node_get_by_name(devlink, parent_name);
1748                 if (IS_ERR(parent))
1749                         return -ENODEV;
1750
1751                 if (parent == devlink_rate) {
1752                         NL_SET_ERR_MSG_MOD(info->extack, "Parent to self is not allowed");
1753                         return -EINVAL;
1754                 }
1755
1756                 if (devlink_rate_is_node(devlink_rate) &&
1757                     devlink_rate_is_parent_node(devlink_rate, parent->parent)) {
1758                         NL_SET_ERR_MSG_MOD(info->extack, "Node is already a parent of parent node.");
1759                         return -EEXIST;
1760                 }
1761
1762                 if (devlink_rate_is_leaf(devlink_rate))
1763                         err = ops->rate_leaf_parent_set(devlink_rate, parent,
1764                                                         devlink_rate->priv, parent->priv,
1765                                                         info->extack);
1766                 else if (devlink_rate_is_node(devlink_rate))
1767                         err = ops->rate_node_parent_set(devlink_rate, parent,
1768                                                         devlink_rate->priv, parent->priv,
1769                                                         info->extack);
1770                 if (err)
1771                         return err;
1772
1773                 refcount_inc(&parent->refcnt);
1774                 devlink_rate->parent = parent;
1775         }
1776
1777         return 0;
1778 }
1779
1780 static int devlink_nl_rate_set(struct devlink_rate *devlink_rate,
1781                                const struct devlink_ops *ops,
1782                                struct genl_info *info)
1783 {
1784         struct nlattr *nla_parent, **attrs = info->attrs;
1785         int err = -EOPNOTSUPP;
1786         u64 rate;
1787
1788         if (attrs[DEVLINK_ATTR_RATE_TX_SHARE]) {
1789                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_SHARE]);
1790                 if (devlink_rate_is_leaf(devlink_rate))
1791                         err = ops->rate_leaf_tx_share_set(devlink_rate, devlink_rate->priv,
1792                                                           rate, info->extack);
1793                 else if (devlink_rate_is_node(devlink_rate))
1794                         err = ops->rate_node_tx_share_set(devlink_rate, devlink_rate->priv,
1795                                                           rate, info->extack);
1796                 if (err)
1797                         return err;
1798                 devlink_rate->tx_share = rate;
1799         }
1800
1801         if (attrs[DEVLINK_ATTR_RATE_TX_MAX]) {
1802                 rate = nla_get_u64(attrs[DEVLINK_ATTR_RATE_TX_MAX]);
1803                 if (devlink_rate_is_leaf(devlink_rate))
1804                         err = ops->rate_leaf_tx_max_set(devlink_rate, devlink_rate->priv,
1805                                                         rate, info->extack);
1806                 else if (devlink_rate_is_node(devlink_rate))
1807                         err = ops->rate_node_tx_max_set(devlink_rate, devlink_rate->priv,
1808                                                         rate, info->extack);
1809                 if (err)
1810                         return err;
1811                 devlink_rate->tx_max = rate;
1812         }
1813
1814         nla_parent = attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME];
1815         if (nla_parent) {
1816                 err = devlink_nl_rate_parent_node_set(devlink_rate, info,
1817                                                       nla_parent);
1818                 if (err)
1819                         return err;
1820         }
1821
1822         return 0;
1823 }
1824
1825 static bool devlink_rate_set_ops_supported(const struct devlink_ops *ops,
1826                                            struct genl_info *info,
1827                                            enum devlink_rate_type type)
1828 {
1829         struct nlattr **attrs = info->attrs;
1830
1831         if (type == DEVLINK_RATE_TYPE_LEAF) {
1832                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_leaf_tx_share_set) {
1833                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the leafs");
1834                         return false;
1835                 }
1836                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_leaf_tx_max_set) {
1837                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the leafs");
1838                         return false;
1839                 }
1840                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1841                     !ops->rate_leaf_parent_set) {
1842                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the leafs");
1843                         return false;
1844                 }
1845         } else if (type == DEVLINK_RATE_TYPE_NODE) {
1846                 if (attrs[DEVLINK_ATTR_RATE_TX_SHARE] && !ops->rate_node_tx_share_set) {
1847                         NL_SET_ERR_MSG_MOD(info->extack, "TX share set isn't supported for the nodes");
1848                         return false;
1849                 }
1850                 if (attrs[DEVLINK_ATTR_RATE_TX_MAX] && !ops->rate_node_tx_max_set) {
1851                         NL_SET_ERR_MSG_MOD(info->extack, "TX max set isn't supported for the nodes");
1852                         return false;
1853                 }
1854                 if (attrs[DEVLINK_ATTR_RATE_PARENT_NODE_NAME] &&
1855                     !ops->rate_node_parent_set) {
1856                         NL_SET_ERR_MSG_MOD(info->extack, "Parent set isn't supported for the nodes");
1857                         return false;
1858                 }
1859         } else {
1860                 WARN(1, "Unknown type of rate object");
1861                 return false;
1862         }
1863
1864         return true;
1865 }
1866
1867 static int devlink_nl_cmd_rate_set_doit(struct sk_buff *skb,
1868                                         struct genl_info *info)
1869 {
1870         struct devlink_rate *devlink_rate = info->user_ptr[1];
1871         struct devlink *devlink = devlink_rate->devlink;
1872         const struct devlink_ops *ops = devlink->ops;
1873         int err;
1874
1875         if (!ops || !devlink_rate_set_ops_supported(ops, info, devlink_rate->type))
1876                 return -EOPNOTSUPP;
1877
1878         err = devlink_nl_rate_set(devlink_rate, ops, info);
1879
1880         if (!err)
1881                 devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
1882         return err;
1883 }
1884
1885 static int devlink_nl_cmd_rate_new_doit(struct sk_buff *skb,
1886                                         struct genl_info *info)
1887 {
1888         struct devlink *devlink = info->user_ptr[0];
1889         struct devlink_rate *rate_node;
1890         const struct devlink_ops *ops;
1891         int err;
1892
1893         ops = devlink->ops;
1894         if (!ops || !ops->rate_node_new || !ops->rate_node_del) {
1895                 NL_SET_ERR_MSG_MOD(info->extack, "Rate nodes aren't supported");
1896                 return -EOPNOTSUPP;
1897         }
1898
1899         if (!devlink_rate_set_ops_supported(ops, info, DEVLINK_RATE_TYPE_NODE))
1900                 return -EOPNOTSUPP;
1901
1902         rate_node = devlink_rate_node_get_from_attrs(devlink, info->attrs);
1903         if (!IS_ERR(rate_node))
1904                 return -EEXIST;
1905         else if (rate_node == ERR_PTR(-EINVAL))
1906                 return -EINVAL;
1907
1908         rate_node = kzalloc(sizeof(*rate_node), GFP_KERNEL);
1909         if (!rate_node)
1910                 return -ENOMEM;
1911
1912         rate_node->devlink = devlink;
1913         rate_node->type = DEVLINK_RATE_TYPE_NODE;
1914         rate_node->name = nla_strdup(info->attrs[DEVLINK_ATTR_RATE_NODE_NAME], GFP_KERNEL);
1915         if (!rate_node->name) {
1916                 err = -ENOMEM;
1917                 goto err_strdup;
1918         }
1919
1920         err = ops->rate_node_new(rate_node, &rate_node->priv, info->extack);
1921         if (err)
1922                 goto err_node_new;
1923
1924         err = devlink_nl_rate_set(rate_node, ops, info);
1925         if (err)
1926                 goto err_rate_set;
1927
1928         refcount_set(&rate_node->refcnt, 1);
1929         list_add(&rate_node->list, &devlink->rate_list);
1930         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
1931         return 0;
1932
1933 err_rate_set:
1934         ops->rate_node_del(rate_node, rate_node->priv, info->extack);
1935 err_node_new:
1936         kfree(rate_node->name);
1937 err_strdup:
1938         kfree(rate_node);
1939         return err;
1940 }
1941
1942 static int devlink_nl_cmd_rate_del_doit(struct sk_buff *skb,
1943                                         struct genl_info *info)
1944 {
1945         struct devlink_rate *rate_node = info->user_ptr[1];
1946         struct devlink *devlink = rate_node->devlink;
1947         const struct devlink_ops *ops = devlink->ops;
1948         int err;
1949
1950         if (refcount_read(&rate_node->refcnt) > 1) {
1951                 NL_SET_ERR_MSG_MOD(info->extack, "Node has children. Cannot delete node.");
1952                 return -EBUSY;
1953         }
1954
1955         devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
1956         err = ops->rate_node_del(rate_node, rate_node->priv, info->extack);
1957         if (rate_node->parent)
1958                 refcount_dec(&rate_node->parent->refcnt);
1959         list_del(&rate_node->list);
1960         kfree(rate_node->name);
1961         kfree(rate_node);
1962         return err;
1963 }
1964
1965 static int devlink_nl_sb_fill(struct sk_buff *msg, struct devlink *devlink,
1966                               struct devlink_sb *devlink_sb,
1967                               enum devlink_command cmd, u32 portid,
1968                               u32 seq, int flags)
1969 {
1970         void *hdr;
1971
1972         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
1973         if (!hdr)
1974                 return -EMSGSIZE;
1975
1976         if (devlink_nl_put_handle(msg, devlink))
1977                 goto nla_put_failure;
1978         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
1979                 goto nla_put_failure;
1980         if (nla_put_u32(msg, DEVLINK_ATTR_SB_SIZE, devlink_sb->size))
1981                 goto nla_put_failure;
1982         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_POOL_COUNT,
1983                         devlink_sb->ingress_pools_count))
1984                 goto nla_put_failure;
1985         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_POOL_COUNT,
1986                         devlink_sb->egress_pools_count))
1987                 goto nla_put_failure;
1988         if (nla_put_u16(msg, DEVLINK_ATTR_SB_INGRESS_TC_COUNT,
1989                         devlink_sb->ingress_tc_count))
1990                 goto nla_put_failure;
1991         if (nla_put_u16(msg, DEVLINK_ATTR_SB_EGRESS_TC_COUNT,
1992                         devlink_sb->egress_tc_count))
1993                 goto nla_put_failure;
1994
1995         genlmsg_end(msg, hdr);
1996         return 0;
1997
1998 nla_put_failure:
1999         genlmsg_cancel(msg, hdr);
2000         return -EMSGSIZE;
2001 }
2002
2003 static int devlink_nl_cmd_sb_get_doit(struct sk_buff *skb,
2004                                       struct genl_info *info)
2005 {
2006         struct devlink *devlink = info->user_ptr[0];
2007         struct devlink_sb *devlink_sb;
2008         struct sk_buff *msg;
2009         int err;
2010
2011         devlink_sb = devlink_sb_get_from_info(devlink, info);
2012         if (IS_ERR(devlink_sb))
2013                 return PTR_ERR(devlink_sb);
2014
2015         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2016         if (!msg)
2017                 return -ENOMEM;
2018
2019         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2020                                  DEVLINK_CMD_SB_NEW,
2021                                  info->snd_portid, info->snd_seq, 0);
2022         if (err) {
2023                 nlmsg_free(msg);
2024                 return err;
2025         }
2026
2027         return genlmsg_reply(msg, info);
2028 }
2029
2030 static int devlink_nl_cmd_sb_get_dumpit(struct sk_buff *msg,
2031                                         struct netlink_callback *cb)
2032 {
2033         struct devlink *devlink;
2034         struct devlink_sb *devlink_sb;
2035         int start = cb->args[0];
2036         unsigned long index;
2037         int idx = 0;
2038         int err;
2039
2040         mutex_lock(&devlink_mutex);
2041         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2042                 if (!devlink_try_get(devlink))
2043                         continue;
2044
2045                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
2046                         goto retry;
2047
2048                 mutex_lock(&devlink->lock);
2049                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2050                         if (idx < start) {
2051                                 idx++;
2052                                 continue;
2053                         }
2054                         err = devlink_nl_sb_fill(msg, devlink, devlink_sb,
2055                                                  DEVLINK_CMD_SB_NEW,
2056                                                  NETLINK_CB(cb->skb).portid,
2057                                                  cb->nlh->nlmsg_seq,
2058                                                  NLM_F_MULTI);
2059                         if (err) {
2060                                 mutex_unlock(&devlink->lock);
2061                                 devlink_put(devlink);
2062                                 goto out;
2063                         }
2064                         idx++;
2065                 }
2066                 mutex_unlock(&devlink->lock);
2067 retry:
2068                 devlink_put(devlink);
2069         }
2070 out:
2071         mutex_unlock(&devlink_mutex);
2072
2073         cb->args[0] = idx;
2074         return msg->len;
2075 }
2076
2077 static int devlink_nl_sb_pool_fill(struct sk_buff *msg, struct devlink *devlink,
2078                                    struct devlink_sb *devlink_sb,
2079                                    u16 pool_index, enum devlink_command cmd,
2080                                    u32 portid, u32 seq, int flags)
2081 {
2082         struct devlink_sb_pool_info pool_info;
2083         void *hdr;
2084         int err;
2085
2086         err = devlink->ops->sb_pool_get(devlink, devlink_sb->index,
2087                                         pool_index, &pool_info);
2088         if (err)
2089                 return err;
2090
2091         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2092         if (!hdr)
2093                 return -EMSGSIZE;
2094
2095         if (devlink_nl_put_handle(msg, devlink))
2096                 goto nla_put_failure;
2097         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2098                 goto nla_put_failure;
2099         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2100                 goto nla_put_failure;
2101         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_info.pool_type))
2102                 goto nla_put_failure;
2103         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_SIZE, pool_info.size))
2104                 goto nla_put_failure;
2105         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE,
2106                        pool_info.threshold_type))
2107                 goto nla_put_failure;
2108         if (nla_put_u32(msg, DEVLINK_ATTR_SB_POOL_CELL_SIZE,
2109                         pool_info.cell_size))
2110                 goto nla_put_failure;
2111
2112         genlmsg_end(msg, hdr);
2113         return 0;
2114
2115 nla_put_failure:
2116         genlmsg_cancel(msg, hdr);
2117         return -EMSGSIZE;
2118 }
2119
2120 static int devlink_nl_cmd_sb_pool_get_doit(struct sk_buff *skb,
2121                                            struct genl_info *info)
2122 {
2123         struct devlink *devlink = info->user_ptr[0];
2124         struct devlink_sb *devlink_sb;
2125         struct sk_buff *msg;
2126         u16 pool_index;
2127         int err;
2128
2129         devlink_sb = devlink_sb_get_from_info(devlink, info);
2130         if (IS_ERR(devlink_sb))
2131                 return PTR_ERR(devlink_sb);
2132
2133         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2134                                                   &pool_index);
2135         if (err)
2136                 return err;
2137
2138         if (!devlink->ops->sb_pool_get)
2139                 return -EOPNOTSUPP;
2140
2141         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2142         if (!msg)
2143                 return -ENOMEM;
2144
2145         err = devlink_nl_sb_pool_fill(msg, devlink, devlink_sb, pool_index,
2146                                       DEVLINK_CMD_SB_POOL_NEW,
2147                                       info->snd_portid, info->snd_seq, 0);
2148         if (err) {
2149                 nlmsg_free(msg);
2150                 return err;
2151         }
2152
2153         return genlmsg_reply(msg, info);
2154 }
2155
2156 static int __sb_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2157                                 struct devlink *devlink,
2158                                 struct devlink_sb *devlink_sb,
2159                                 u32 portid, u32 seq)
2160 {
2161         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2162         u16 pool_index;
2163         int err;
2164
2165         for (pool_index = 0; pool_index < pool_count; pool_index++) {
2166                 if (*p_idx < start) {
2167                         (*p_idx)++;
2168                         continue;
2169                 }
2170                 err = devlink_nl_sb_pool_fill(msg, devlink,
2171                                               devlink_sb,
2172                                               pool_index,
2173                                               DEVLINK_CMD_SB_POOL_NEW,
2174                                               portid, seq, NLM_F_MULTI);
2175                 if (err)
2176                         return err;
2177                 (*p_idx)++;
2178         }
2179         return 0;
2180 }
2181
2182 static int devlink_nl_cmd_sb_pool_get_dumpit(struct sk_buff *msg,
2183                                              struct netlink_callback *cb)
2184 {
2185         struct devlink *devlink;
2186         struct devlink_sb *devlink_sb;
2187         int start = cb->args[0];
2188         unsigned long index;
2189         int idx = 0;
2190         int err = 0;
2191
2192         mutex_lock(&devlink_mutex);
2193         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2194                 if (!devlink_try_get(devlink))
2195                         continue;
2196
2197                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2198                     !devlink->ops->sb_pool_get)
2199                         goto retry;
2200
2201                 mutex_lock(&devlink->lock);
2202                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2203                         err = __sb_pool_get_dumpit(msg, start, &idx, devlink,
2204                                                    devlink_sb,
2205                                                    NETLINK_CB(cb->skb).portid,
2206                                                    cb->nlh->nlmsg_seq);
2207                         if (err == -EOPNOTSUPP) {
2208                                 err = 0;
2209                         } else if (err) {
2210                                 mutex_unlock(&devlink->lock);
2211                                 devlink_put(devlink);
2212                                 goto out;
2213                         }
2214                 }
2215                 mutex_unlock(&devlink->lock);
2216 retry:
2217                 devlink_put(devlink);
2218         }
2219 out:
2220         mutex_unlock(&devlink_mutex);
2221
2222         if (err != -EMSGSIZE)
2223                 return err;
2224
2225         cb->args[0] = idx;
2226         return msg->len;
2227 }
2228
2229 static int devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index,
2230                                u16 pool_index, u32 size,
2231                                enum devlink_sb_threshold_type threshold_type,
2232                                struct netlink_ext_ack *extack)
2233
2234 {
2235         const struct devlink_ops *ops = devlink->ops;
2236
2237         if (ops->sb_pool_set)
2238                 return ops->sb_pool_set(devlink, sb_index, pool_index,
2239                                         size, threshold_type, extack);
2240         return -EOPNOTSUPP;
2241 }
2242
2243 static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
2244                                            struct genl_info *info)
2245 {
2246         struct devlink *devlink = info->user_ptr[0];
2247         enum devlink_sb_threshold_type threshold_type;
2248         struct devlink_sb *devlink_sb;
2249         u16 pool_index;
2250         u32 size;
2251         int err;
2252
2253         devlink_sb = devlink_sb_get_from_info(devlink, info);
2254         if (IS_ERR(devlink_sb))
2255                 return PTR_ERR(devlink_sb);
2256
2257         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2258                                                   &pool_index);
2259         if (err)
2260                 return err;
2261
2262         err = devlink_sb_th_type_get_from_info(info, &threshold_type);
2263         if (err)
2264                 return err;
2265
2266         if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
2267                 return -EINVAL;
2268
2269         size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
2270         return devlink_sb_pool_set(devlink, devlink_sb->index,
2271                                    pool_index, size, threshold_type,
2272                                    info->extack);
2273 }
2274
2275 static int devlink_nl_sb_port_pool_fill(struct sk_buff *msg,
2276                                         struct devlink *devlink,
2277                                         struct devlink_port *devlink_port,
2278                                         struct devlink_sb *devlink_sb,
2279                                         u16 pool_index,
2280                                         enum devlink_command cmd,
2281                                         u32 portid, u32 seq, int flags)
2282 {
2283         const struct devlink_ops *ops = devlink->ops;
2284         u32 threshold;
2285         void *hdr;
2286         int err;
2287
2288         err = ops->sb_port_pool_get(devlink_port, devlink_sb->index,
2289                                     pool_index, &threshold);
2290         if (err)
2291                 return err;
2292
2293         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2294         if (!hdr)
2295                 return -EMSGSIZE;
2296
2297         if (devlink_nl_put_handle(msg, devlink))
2298                 goto nla_put_failure;
2299         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2300                 goto nla_put_failure;
2301         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2302                 goto nla_put_failure;
2303         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2304                 goto nla_put_failure;
2305         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2306                 goto nla_put_failure;
2307
2308         if (ops->sb_occ_port_pool_get) {
2309                 u32 cur;
2310                 u32 max;
2311
2312                 err = ops->sb_occ_port_pool_get(devlink_port, devlink_sb->index,
2313                                                 pool_index, &cur, &max);
2314                 if (err && err != -EOPNOTSUPP)
2315                         goto sb_occ_get_failure;
2316                 if (!err) {
2317                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2318                                 goto nla_put_failure;
2319                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2320                                 goto nla_put_failure;
2321                 }
2322         }
2323
2324         genlmsg_end(msg, hdr);
2325         return 0;
2326
2327 nla_put_failure:
2328         err = -EMSGSIZE;
2329 sb_occ_get_failure:
2330         genlmsg_cancel(msg, hdr);
2331         return err;
2332 }
2333
2334 static int devlink_nl_cmd_sb_port_pool_get_doit(struct sk_buff *skb,
2335                                                 struct genl_info *info)
2336 {
2337         struct devlink_port *devlink_port = info->user_ptr[1];
2338         struct devlink *devlink = devlink_port->devlink;
2339         struct devlink_sb *devlink_sb;
2340         struct sk_buff *msg;
2341         u16 pool_index;
2342         int err;
2343
2344         devlink_sb = devlink_sb_get_from_info(devlink, info);
2345         if (IS_ERR(devlink_sb))
2346                 return PTR_ERR(devlink_sb);
2347
2348         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2349                                                   &pool_index);
2350         if (err)
2351                 return err;
2352
2353         if (!devlink->ops->sb_port_pool_get)
2354                 return -EOPNOTSUPP;
2355
2356         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2357         if (!msg)
2358                 return -ENOMEM;
2359
2360         err = devlink_nl_sb_port_pool_fill(msg, devlink, devlink_port,
2361                                            devlink_sb, pool_index,
2362                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2363                                            info->snd_portid, info->snd_seq, 0);
2364         if (err) {
2365                 nlmsg_free(msg);
2366                 return err;
2367         }
2368
2369         return genlmsg_reply(msg, info);
2370 }
2371
2372 static int __sb_port_pool_get_dumpit(struct sk_buff *msg, int start, int *p_idx,
2373                                      struct devlink *devlink,
2374                                      struct devlink_sb *devlink_sb,
2375                                      u32 portid, u32 seq)
2376 {
2377         struct devlink_port *devlink_port;
2378         u16 pool_count = devlink_sb_pool_count(devlink_sb);
2379         u16 pool_index;
2380         int err;
2381
2382         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2383                 for (pool_index = 0; pool_index < pool_count; pool_index++) {
2384                         if (*p_idx < start) {
2385                                 (*p_idx)++;
2386                                 continue;
2387                         }
2388                         err = devlink_nl_sb_port_pool_fill(msg, devlink,
2389                                                            devlink_port,
2390                                                            devlink_sb,
2391                                                            pool_index,
2392                                                            DEVLINK_CMD_SB_PORT_POOL_NEW,
2393                                                            portid, seq,
2394                                                            NLM_F_MULTI);
2395                         if (err)
2396                                 return err;
2397                         (*p_idx)++;
2398                 }
2399         }
2400         return 0;
2401 }
2402
2403 static int devlink_nl_cmd_sb_port_pool_get_dumpit(struct sk_buff *msg,
2404                                                   struct netlink_callback *cb)
2405 {
2406         struct devlink *devlink;
2407         struct devlink_sb *devlink_sb;
2408         int start = cb->args[0];
2409         unsigned long index;
2410         int idx = 0;
2411         int err = 0;
2412
2413         mutex_lock(&devlink_mutex);
2414         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2415                 if (!devlink_try_get(devlink))
2416                         continue;
2417
2418                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2419                     !devlink->ops->sb_port_pool_get)
2420                         goto retry;
2421
2422                 mutex_lock(&devlink->lock);
2423                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2424                         err = __sb_port_pool_get_dumpit(msg, start, &idx,
2425                                                         devlink, devlink_sb,
2426                                                         NETLINK_CB(cb->skb).portid,
2427                                                         cb->nlh->nlmsg_seq);
2428                         if (err == -EOPNOTSUPP) {
2429                                 err = 0;
2430                         } else if (err) {
2431                                 mutex_unlock(&devlink->lock);
2432                                 devlink_put(devlink);
2433                                 goto out;
2434                         }
2435                 }
2436                 mutex_unlock(&devlink->lock);
2437 retry:
2438                 devlink_put(devlink);
2439         }
2440 out:
2441         mutex_unlock(&devlink_mutex);
2442
2443         if (err != -EMSGSIZE)
2444                 return err;
2445
2446         cb->args[0] = idx;
2447         return msg->len;
2448 }
2449
2450 static int devlink_sb_port_pool_set(struct devlink_port *devlink_port,
2451                                     unsigned int sb_index, u16 pool_index,
2452                                     u32 threshold,
2453                                     struct netlink_ext_ack *extack)
2454
2455 {
2456         const struct devlink_ops *ops = devlink_port->devlink->ops;
2457
2458         if (ops->sb_port_pool_set)
2459                 return ops->sb_port_pool_set(devlink_port, sb_index,
2460                                              pool_index, threshold, extack);
2461         return -EOPNOTSUPP;
2462 }
2463
2464 static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
2465                                                 struct genl_info *info)
2466 {
2467         struct devlink_port *devlink_port = info->user_ptr[1];
2468         struct devlink *devlink = info->user_ptr[0];
2469         struct devlink_sb *devlink_sb;
2470         u16 pool_index;
2471         u32 threshold;
2472         int err;
2473
2474         devlink_sb = devlink_sb_get_from_info(devlink, info);
2475         if (IS_ERR(devlink_sb))
2476                 return PTR_ERR(devlink_sb);
2477
2478         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2479                                                   &pool_index);
2480         if (err)
2481                 return err;
2482
2483         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2484                 return -EINVAL;
2485
2486         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2487         return devlink_sb_port_pool_set(devlink_port, devlink_sb->index,
2488                                         pool_index, threshold, info->extack);
2489 }
2490
2491 static int
2492 devlink_nl_sb_tc_pool_bind_fill(struct sk_buff *msg, struct devlink *devlink,
2493                                 struct devlink_port *devlink_port,
2494                                 struct devlink_sb *devlink_sb, u16 tc_index,
2495                                 enum devlink_sb_pool_type pool_type,
2496                                 enum devlink_command cmd,
2497                                 u32 portid, u32 seq, int flags)
2498 {
2499         const struct devlink_ops *ops = devlink->ops;
2500         u16 pool_index;
2501         u32 threshold;
2502         void *hdr;
2503         int err;
2504
2505         err = ops->sb_tc_pool_bind_get(devlink_port, devlink_sb->index,
2506                                        tc_index, pool_type,
2507                                        &pool_index, &threshold);
2508         if (err)
2509                 return err;
2510
2511         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2512         if (!hdr)
2513                 return -EMSGSIZE;
2514
2515         if (devlink_nl_put_handle(msg, devlink))
2516                 goto nla_put_failure;
2517         if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, devlink_port->index))
2518                 goto nla_put_failure;
2519         if (nla_put_u32(msg, DEVLINK_ATTR_SB_INDEX, devlink_sb->index))
2520                 goto nla_put_failure;
2521         if (nla_put_u16(msg, DEVLINK_ATTR_SB_TC_INDEX, tc_index))
2522                 goto nla_put_failure;
2523         if (nla_put_u8(msg, DEVLINK_ATTR_SB_POOL_TYPE, pool_type))
2524                 goto nla_put_failure;
2525         if (nla_put_u16(msg, DEVLINK_ATTR_SB_POOL_INDEX, pool_index))
2526                 goto nla_put_failure;
2527         if (nla_put_u32(msg, DEVLINK_ATTR_SB_THRESHOLD, threshold))
2528                 goto nla_put_failure;
2529
2530         if (ops->sb_occ_tc_port_bind_get) {
2531                 u32 cur;
2532                 u32 max;
2533
2534                 err = ops->sb_occ_tc_port_bind_get(devlink_port,
2535                                                    devlink_sb->index,
2536                                                    tc_index, pool_type,
2537                                                    &cur, &max);
2538                 if (err && err != -EOPNOTSUPP)
2539                         return err;
2540                 if (!err) {
2541                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_CUR, cur))
2542                                 goto nla_put_failure;
2543                         if (nla_put_u32(msg, DEVLINK_ATTR_SB_OCC_MAX, max))
2544                                 goto nla_put_failure;
2545                 }
2546         }
2547
2548         genlmsg_end(msg, hdr);
2549         return 0;
2550
2551 nla_put_failure:
2552         genlmsg_cancel(msg, hdr);
2553         return -EMSGSIZE;
2554 }
2555
2556 static int devlink_nl_cmd_sb_tc_pool_bind_get_doit(struct sk_buff *skb,
2557                                                    struct genl_info *info)
2558 {
2559         struct devlink_port *devlink_port = info->user_ptr[1];
2560         struct devlink *devlink = devlink_port->devlink;
2561         struct devlink_sb *devlink_sb;
2562         struct sk_buff *msg;
2563         enum devlink_sb_pool_type pool_type;
2564         u16 tc_index;
2565         int err;
2566
2567         devlink_sb = devlink_sb_get_from_info(devlink, info);
2568         if (IS_ERR(devlink_sb))
2569                 return PTR_ERR(devlink_sb);
2570
2571         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2572         if (err)
2573                 return err;
2574
2575         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2576                                                 pool_type, &tc_index);
2577         if (err)
2578                 return err;
2579
2580         if (!devlink->ops->sb_tc_pool_bind_get)
2581                 return -EOPNOTSUPP;
2582
2583         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2584         if (!msg)
2585                 return -ENOMEM;
2586
2587         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink, devlink_port,
2588                                               devlink_sb, tc_index, pool_type,
2589                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2590                                               info->snd_portid,
2591                                               info->snd_seq, 0);
2592         if (err) {
2593                 nlmsg_free(msg);
2594                 return err;
2595         }
2596
2597         return genlmsg_reply(msg, info);
2598 }
2599
2600 static int __sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
2601                                         int start, int *p_idx,
2602                                         struct devlink *devlink,
2603                                         struct devlink_sb *devlink_sb,
2604                                         u32 portid, u32 seq)
2605 {
2606         struct devlink_port *devlink_port;
2607         u16 tc_index;
2608         int err;
2609
2610         list_for_each_entry(devlink_port, &devlink->port_list, list) {
2611                 for (tc_index = 0;
2612                      tc_index < devlink_sb->ingress_tc_count; tc_index++) {
2613                         if (*p_idx < start) {
2614                                 (*p_idx)++;
2615                                 continue;
2616                         }
2617                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
2618                                                               devlink_port,
2619                                                               devlink_sb,
2620                                                               tc_index,
2621                                                               DEVLINK_SB_POOL_TYPE_INGRESS,
2622                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2623                                                               portid, seq,
2624                                                               NLM_F_MULTI);
2625                         if (err)
2626                                 return err;
2627                         (*p_idx)++;
2628                 }
2629                 for (tc_index = 0;
2630                      tc_index < devlink_sb->egress_tc_count; tc_index++) {
2631                         if (*p_idx < start) {
2632                                 (*p_idx)++;
2633                                 continue;
2634                         }
2635                         err = devlink_nl_sb_tc_pool_bind_fill(msg, devlink,
2636                                                               devlink_port,
2637                                                               devlink_sb,
2638                                                               tc_index,
2639                                                               DEVLINK_SB_POOL_TYPE_EGRESS,
2640                                                               DEVLINK_CMD_SB_TC_POOL_BIND_NEW,
2641                                                               portid, seq,
2642                                                               NLM_F_MULTI);
2643                         if (err)
2644                                 return err;
2645                         (*p_idx)++;
2646                 }
2647         }
2648         return 0;
2649 }
2650
2651 static int
2652 devlink_nl_cmd_sb_tc_pool_bind_get_dumpit(struct sk_buff *msg,
2653                                           struct netlink_callback *cb)
2654 {
2655         struct devlink *devlink;
2656         struct devlink_sb *devlink_sb;
2657         int start = cb->args[0];
2658         unsigned long index;
2659         int idx = 0;
2660         int err = 0;
2661
2662         mutex_lock(&devlink_mutex);
2663         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
2664                 if (!devlink_try_get(devlink))
2665                         continue;
2666
2667                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)) ||
2668                     !devlink->ops->sb_tc_pool_bind_get)
2669                         goto retry;
2670
2671                 mutex_lock(&devlink->lock);
2672                 list_for_each_entry(devlink_sb, &devlink->sb_list, list) {
2673                         err = __sb_tc_pool_bind_get_dumpit(msg, start, &idx,
2674                                                            devlink,
2675                                                            devlink_sb,
2676                                                            NETLINK_CB(cb->skb).portid,
2677                                                            cb->nlh->nlmsg_seq);
2678                         if (err == -EOPNOTSUPP) {
2679                                 err = 0;
2680                         } else if (err) {
2681                                 mutex_unlock(&devlink->lock);
2682                                 devlink_put(devlink);
2683                                 goto out;
2684                         }
2685                 }
2686                 mutex_unlock(&devlink->lock);
2687 retry:
2688                 devlink_put(devlink);
2689         }
2690 out:
2691         mutex_unlock(&devlink_mutex);
2692
2693         if (err != -EMSGSIZE)
2694                 return err;
2695
2696         cb->args[0] = idx;
2697         return msg->len;
2698 }
2699
2700 static int devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
2701                                        unsigned int sb_index, u16 tc_index,
2702                                        enum devlink_sb_pool_type pool_type,
2703                                        u16 pool_index, u32 threshold,
2704                                        struct netlink_ext_ack *extack)
2705
2706 {
2707         const struct devlink_ops *ops = devlink_port->devlink->ops;
2708
2709         if (ops->sb_tc_pool_bind_set)
2710                 return ops->sb_tc_pool_bind_set(devlink_port, sb_index,
2711                                                 tc_index, pool_type,
2712                                                 pool_index, threshold, extack);
2713         return -EOPNOTSUPP;
2714 }
2715
2716 static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
2717                                                    struct genl_info *info)
2718 {
2719         struct devlink_port *devlink_port = info->user_ptr[1];
2720         struct devlink *devlink = info->user_ptr[0];
2721         enum devlink_sb_pool_type pool_type;
2722         struct devlink_sb *devlink_sb;
2723         u16 tc_index;
2724         u16 pool_index;
2725         u32 threshold;
2726         int err;
2727
2728         devlink_sb = devlink_sb_get_from_info(devlink, info);
2729         if (IS_ERR(devlink_sb))
2730                 return PTR_ERR(devlink_sb);
2731
2732         err = devlink_sb_pool_type_get_from_info(info, &pool_type);
2733         if (err)
2734                 return err;
2735
2736         err = devlink_sb_tc_index_get_from_info(devlink_sb, info,
2737                                                 pool_type, &tc_index);
2738         if (err)
2739                 return err;
2740
2741         err = devlink_sb_pool_index_get_from_info(devlink_sb, info,
2742                                                   &pool_index);
2743         if (err)
2744                 return err;
2745
2746         if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
2747                 return -EINVAL;
2748
2749         threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
2750         return devlink_sb_tc_pool_bind_set(devlink_port, devlink_sb->index,
2751                                            tc_index, pool_type,
2752                                            pool_index, threshold, info->extack);
2753 }
2754
2755 static int devlink_nl_cmd_sb_occ_snapshot_doit(struct sk_buff *skb,
2756                                                struct genl_info *info)
2757 {
2758         struct devlink *devlink = info->user_ptr[0];
2759         const struct devlink_ops *ops = devlink->ops;
2760         struct devlink_sb *devlink_sb;
2761
2762         devlink_sb = devlink_sb_get_from_info(devlink, info);
2763         if (IS_ERR(devlink_sb))
2764                 return PTR_ERR(devlink_sb);
2765
2766         if (ops->sb_occ_snapshot)
2767                 return ops->sb_occ_snapshot(devlink, devlink_sb->index);
2768         return -EOPNOTSUPP;
2769 }
2770
2771 static int devlink_nl_cmd_sb_occ_max_clear_doit(struct sk_buff *skb,
2772                                                 struct genl_info *info)
2773 {
2774         struct devlink *devlink = info->user_ptr[0];
2775         const struct devlink_ops *ops = devlink->ops;
2776         struct devlink_sb *devlink_sb;
2777
2778         devlink_sb = devlink_sb_get_from_info(devlink, info);
2779         if (IS_ERR(devlink_sb))
2780                 return PTR_ERR(devlink_sb);
2781
2782         if (ops->sb_occ_max_clear)
2783                 return ops->sb_occ_max_clear(devlink, devlink_sb->index);
2784         return -EOPNOTSUPP;
2785 }
2786
2787 static int devlink_nl_eswitch_fill(struct sk_buff *msg, struct devlink *devlink,
2788                                    enum devlink_command cmd, u32 portid,
2789                                    u32 seq, int flags)
2790 {
2791         const struct devlink_ops *ops = devlink->ops;
2792         enum devlink_eswitch_encap_mode encap_mode;
2793         u8 inline_mode;
2794         void *hdr;
2795         int err = 0;
2796         u16 mode;
2797
2798         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
2799         if (!hdr)
2800                 return -EMSGSIZE;
2801
2802         err = devlink_nl_put_handle(msg, devlink);
2803         if (err)
2804                 goto nla_put_failure;
2805
2806         if (ops->eswitch_mode_get) {
2807                 err = ops->eswitch_mode_get(devlink, &mode);
2808                 if (err)
2809                         goto nla_put_failure;
2810                 err = nla_put_u16(msg, DEVLINK_ATTR_ESWITCH_MODE, mode);
2811                 if (err)
2812                         goto nla_put_failure;
2813         }
2814
2815         if (ops->eswitch_inline_mode_get) {
2816                 err = ops->eswitch_inline_mode_get(devlink, &inline_mode);
2817                 if (err)
2818                         goto nla_put_failure;
2819                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_INLINE_MODE,
2820                                  inline_mode);
2821                 if (err)
2822                         goto nla_put_failure;
2823         }
2824
2825         if (ops->eswitch_encap_mode_get) {
2826                 err = ops->eswitch_encap_mode_get(devlink, &encap_mode);
2827                 if (err)
2828                         goto nla_put_failure;
2829                 err = nla_put_u8(msg, DEVLINK_ATTR_ESWITCH_ENCAP_MODE, encap_mode);
2830                 if (err)
2831                         goto nla_put_failure;
2832         }
2833
2834         genlmsg_end(msg, hdr);
2835         return 0;
2836
2837 nla_put_failure:
2838         genlmsg_cancel(msg, hdr);
2839         return err;
2840 }
2841
2842 static int devlink_nl_cmd_eswitch_get_doit(struct sk_buff *skb,
2843                                            struct genl_info *info)
2844 {
2845         struct devlink *devlink = info->user_ptr[0];
2846         struct sk_buff *msg;
2847         int err;
2848
2849         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
2850         if (!msg)
2851                 return -ENOMEM;
2852
2853         err = devlink_nl_eswitch_fill(msg, devlink, DEVLINK_CMD_ESWITCH_GET,
2854                                       info->snd_portid, info->snd_seq, 0);
2855
2856         if (err) {
2857                 nlmsg_free(msg);
2858                 return err;
2859         }
2860
2861         return genlmsg_reply(msg, info);
2862 }
2863
2864 static int devlink_rate_nodes_check(struct devlink *devlink, u16 mode,
2865                                     struct netlink_ext_ack *extack)
2866 {
2867         struct devlink_rate *devlink_rate;
2868
2869         /* Take the lock to sync with devlink_rate_nodes_destroy() */
2870         mutex_lock(&devlink->lock);
2871         list_for_each_entry(devlink_rate, &devlink->rate_list, list)
2872                 if (devlink_rate_is_node(devlink_rate)) {
2873                         mutex_unlock(&devlink->lock);
2874                         NL_SET_ERR_MSG_MOD(extack, "Rate node(s) exists.");
2875                         return -EBUSY;
2876                 }
2877         mutex_unlock(&devlink->lock);
2878         return 0;
2879 }
2880
2881 static int devlink_nl_cmd_eswitch_set_doit(struct sk_buff *skb,
2882                                            struct genl_info *info)
2883 {
2884         struct devlink *devlink = info->user_ptr[0];
2885         const struct devlink_ops *ops = devlink->ops;
2886         enum devlink_eswitch_encap_mode encap_mode;
2887         u8 inline_mode;
2888         int err = 0;
2889         u16 mode;
2890
2891         if (info->attrs[DEVLINK_ATTR_ESWITCH_MODE]) {
2892                 if (!ops->eswitch_mode_set)
2893                         return -EOPNOTSUPP;
2894                 mode = nla_get_u16(info->attrs[DEVLINK_ATTR_ESWITCH_MODE]);
2895                 err = devlink_rate_nodes_check(devlink, mode, info->extack);
2896                 if (err)
2897                         return err;
2898                 err = ops->eswitch_mode_set(devlink, mode, info->extack);
2899                 if (err)
2900                         return err;
2901         }
2902
2903         if (info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]) {
2904                 if (!ops->eswitch_inline_mode_set)
2905                         return -EOPNOTSUPP;
2906                 inline_mode = nla_get_u8(
2907                                 info->attrs[DEVLINK_ATTR_ESWITCH_INLINE_MODE]);
2908                 err = ops->eswitch_inline_mode_set(devlink, inline_mode,
2909                                                    info->extack);
2910                 if (err)
2911                         return err;
2912         }
2913
2914         if (info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]) {
2915                 if (!ops->eswitch_encap_mode_set)
2916                         return -EOPNOTSUPP;
2917                 encap_mode = nla_get_u8(info->attrs[DEVLINK_ATTR_ESWITCH_ENCAP_MODE]);
2918                 err = ops->eswitch_encap_mode_set(devlink, encap_mode,
2919                                                   info->extack);
2920                 if (err)
2921                         return err;
2922         }
2923
2924         return 0;
2925 }
2926
2927 int devlink_dpipe_match_put(struct sk_buff *skb,
2928                             struct devlink_dpipe_match *match)
2929 {
2930         struct devlink_dpipe_header *header = match->header;
2931         struct devlink_dpipe_field *field = &header->fields[match->field_id];
2932         struct nlattr *match_attr;
2933
2934         match_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_MATCH);
2935         if (!match_attr)
2936                 return -EMSGSIZE;
2937
2938         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_MATCH_TYPE, match->type) ||
2939             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, match->header_index) ||
2940             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2941             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2942             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2943                 goto nla_put_failure;
2944
2945         nla_nest_end(skb, match_attr);
2946         return 0;
2947
2948 nla_put_failure:
2949         nla_nest_cancel(skb, match_attr);
2950         return -EMSGSIZE;
2951 }
2952 EXPORT_SYMBOL_GPL(devlink_dpipe_match_put);
2953
2954 static int devlink_dpipe_matches_put(struct devlink_dpipe_table *table,
2955                                      struct sk_buff *skb)
2956 {
2957         struct nlattr *matches_attr;
2958
2959         matches_attr = nla_nest_start_noflag(skb,
2960                                              DEVLINK_ATTR_DPIPE_TABLE_MATCHES);
2961         if (!matches_attr)
2962                 return -EMSGSIZE;
2963
2964         if (table->table_ops->matches_dump(table->priv, skb))
2965                 goto nla_put_failure;
2966
2967         nla_nest_end(skb, matches_attr);
2968         return 0;
2969
2970 nla_put_failure:
2971         nla_nest_cancel(skb, matches_attr);
2972         return -EMSGSIZE;
2973 }
2974
2975 int devlink_dpipe_action_put(struct sk_buff *skb,
2976                              struct devlink_dpipe_action *action)
2977 {
2978         struct devlink_dpipe_header *header = action->header;
2979         struct devlink_dpipe_field *field = &header->fields[action->field_id];
2980         struct nlattr *action_attr;
2981
2982         action_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ACTION);
2983         if (!action_attr)
2984                 return -EMSGSIZE;
2985
2986         if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_ACTION_TYPE, action->type) ||
2987             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_INDEX, action->header_index) ||
2988             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
2989             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
2990             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
2991                 goto nla_put_failure;
2992
2993         nla_nest_end(skb, action_attr);
2994         return 0;
2995
2996 nla_put_failure:
2997         nla_nest_cancel(skb, action_attr);
2998         return -EMSGSIZE;
2999 }
3000 EXPORT_SYMBOL_GPL(devlink_dpipe_action_put);
3001
3002 static int devlink_dpipe_actions_put(struct devlink_dpipe_table *table,
3003                                      struct sk_buff *skb)
3004 {
3005         struct nlattr *actions_attr;
3006
3007         actions_attr = nla_nest_start_noflag(skb,
3008                                              DEVLINK_ATTR_DPIPE_TABLE_ACTIONS);
3009         if (!actions_attr)
3010                 return -EMSGSIZE;
3011
3012         if (table->table_ops->actions_dump(table->priv, skb))
3013                 goto nla_put_failure;
3014
3015         nla_nest_end(skb, actions_attr);
3016         return 0;
3017
3018 nla_put_failure:
3019         nla_nest_cancel(skb, actions_attr);
3020         return -EMSGSIZE;
3021 }
3022
3023 static int devlink_dpipe_table_put(struct sk_buff *skb,
3024                                    struct devlink_dpipe_table *table)
3025 {
3026         struct nlattr *table_attr;
3027         u64 table_size;
3028
3029         table_size = table->table_ops->size_get(table->priv);
3030         table_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLE);
3031         if (!table_attr)
3032                 return -EMSGSIZE;
3033
3034         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_TABLE_NAME, table->name) ||
3035             nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_SIZE, table_size,
3036                               DEVLINK_ATTR_PAD))
3037                 goto nla_put_failure;
3038         if (nla_put_u8(skb, DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED,
3039                        table->counters_enabled))
3040                 goto nla_put_failure;
3041
3042         if (table->resource_valid) {
3043                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_ID,
3044                                       table->resource_id, DEVLINK_ATTR_PAD) ||
3045                     nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_TABLE_RESOURCE_UNITS,
3046                                       table->resource_units, DEVLINK_ATTR_PAD))
3047                         goto nla_put_failure;
3048         }
3049         if (devlink_dpipe_matches_put(table, skb))
3050                 goto nla_put_failure;
3051
3052         if (devlink_dpipe_actions_put(table, skb))
3053                 goto nla_put_failure;
3054
3055         nla_nest_end(skb, table_attr);
3056         return 0;
3057
3058 nla_put_failure:
3059         nla_nest_cancel(skb, table_attr);
3060         return -EMSGSIZE;
3061 }
3062
3063 static int devlink_dpipe_send_and_alloc_skb(struct sk_buff **pskb,
3064                                             struct genl_info *info)
3065 {
3066         int err;
3067
3068         if (*pskb) {
3069                 err = genlmsg_reply(*pskb, info);
3070                 if (err)
3071                         return err;
3072         }
3073         *pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
3074         if (!*pskb)
3075                 return -ENOMEM;
3076         return 0;
3077 }
3078
3079 static int devlink_dpipe_tables_fill(struct genl_info *info,
3080                                      enum devlink_command cmd, int flags,
3081                                      struct list_head *dpipe_tables,
3082                                      const char *table_name)
3083 {
3084         struct devlink *devlink = info->user_ptr[0];
3085         struct devlink_dpipe_table *table;
3086         struct nlattr *tables_attr;
3087         struct sk_buff *skb = NULL;
3088         struct nlmsghdr *nlh;
3089         bool incomplete;
3090         void *hdr;
3091         int i;
3092         int err;
3093
3094         table = list_first_entry(dpipe_tables,
3095                                  struct devlink_dpipe_table, list);
3096 start_again:
3097         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3098         if (err)
3099                 return err;
3100
3101         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3102                           &devlink_nl_family, NLM_F_MULTI, cmd);
3103         if (!hdr) {
3104                 nlmsg_free(skb);
3105                 return -EMSGSIZE;
3106         }
3107
3108         if (devlink_nl_put_handle(skb, devlink))
3109                 goto nla_put_failure;
3110         tables_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_TABLES);
3111         if (!tables_attr)
3112                 goto nla_put_failure;
3113
3114         i = 0;
3115         incomplete = false;
3116         list_for_each_entry_from(table, dpipe_tables, list) {
3117                 if (!table_name) {
3118                         err = devlink_dpipe_table_put(skb, table);
3119                         if (err) {
3120                                 if (!i)
3121                                         goto err_table_put;
3122                                 incomplete = true;
3123                                 break;
3124                         }
3125                 } else {
3126                         if (!strcmp(table->name, table_name)) {
3127                                 err = devlink_dpipe_table_put(skb, table);
3128                                 if (err)
3129                                         break;
3130                         }
3131                 }
3132                 i++;
3133         }
3134
3135         nla_nest_end(skb, tables_attr);
3136         genlmsg_end(skb, hdr);
3137         if (incomplete)
3138                 goto start_again;
3139
3140 send_done:
3141         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3142                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3143         if (!nlh) {
3144                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3145                 if (err)
3146                         return err;
3147                 goto send_done;
3148         }
3149
3150         return genlmsg_reply(skb, info);
3151
3152 nla_put_failure:
3153         err = -EMSGSIZE;
3154 err_table_put:
3155         nlmsg_free(skb);
3156         return err;
3157 }
3158
3159 static int devlink_nl_cmd_dpipe_table_get(struct sk_buff *skb,
3160                                           struct genl_info *info)
3161 {
3162         struct devlink *devlink = info->user_ptr[0];
3163         const char *table_name =  NULL;
3164
3165         if (info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3166                 table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3167
3168         return devlink_dpipe_tables_fill(info, DEVLINK_CMD_DPIPE_TABLE_GET, 0,
3169                                          &devlink->dpipe_table_list,
3170                                          table_name);
3171 }
3172
3173 static int devlink_dpipe_value_put(struct sk_buff *skb,
3174                                    struct devlink_dpipe_value *value)
3175 {
3176         if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE,
3177                     value->value_size, value->value))
3178                 return -EMSGSIZE;
3179         if (value->mask)
3180                 if (nla_put(skb, DEVLINK_ATTR_DPIPE_VALUE_MASK,
3181                             value->value_size, value->mask))
3182                         return -EMSGSIZE;
3183         if (value->mapping_valid)
3184                 if (nla_put_u32(skb, DEVLINK_ATTR_DPIPE_VALUE_MAPPING,
3185                                 value->mapping_value))
3186                         return -EMSGSIZE;
3187         return 0;
3188 }
3189
3190 static int devlink_dpipe_action_value_put(struct sk_buff *skb,
3191                                           struct devlink_dpipe_value *value)
3192 {
3193         if (!value->action)
3194                 return -EINVAL;
3195         if (devlink_dpipe_action_put(skb, value->action))
3196                 return -EMSGSIZE;
3197         if (devlink_dpipe_value_put(skb, value))
3198                 return -EMSGSIZE;
3199         return 0;
3200 }
3201
3202 static int devlink_dpipe_action_values_put(struct sk_buff *skb,
3203                                            struct devlink_dpipe_value *values,
3204                                            unsigned int values_count)
3205 {
3206         struct nlattr *action_attr;
3207         int i;
3208         int err;
3209
3210         for (i = 0; i < values_count; i++) {
3211                 action_attr = nla_nest_start_noflag(skb,
3212                                                     DEVLINK_ATTR_DPIPE_ACTION_VALUE);
3213                 if (!action_attr)
3214                         return -EMSGSIZE;
3215                 err = devlink_dpipe_action_value_put(skb, &values[i]);
3216                 if (err)
3217                         goto err_action_value_put;
3218                 nla_nest_end(skb, action_attr);
3219         }
3220         return 0;
3221
3222 err_action_value_put:
3223         nla_nest_cancel(skb, action_attr);
3224         return err;
3225 }
3226
3227 static int devlink_dpipe_match_value_put(struct sk_buff *skb,
3228                                          struct devlink_dpipe_value *value)
3229 {
3230         if (!value->match)
3231                 return -EINVAL;
3232         if (devlink_dpipe_match_put(skb, value->match))
3233                 return -EMSGSIZE;
3234         if (devlink_dpipe_value_put(skb, value))
3235                 return -EMSGSIZE;
3236         return 0;
3237 }
3238
3239 static int devlink_dpipe_match_values_put(struct sk_buff *skb,
3240                                           struct devlink_dpipe_value *values,
3241                                           unsigned int values_count)
3242 {
3243         struct nlattr *match_attr;
3244         int i;
3245         int err;
3246
3247         for (i = 0; i < values_count; i++) {
3248                 match_attr = nla_nest_start_noflag(skb,
3249                                                    DEVLINK_ATTR_DPIPE_MATCH_VALUE);
3250                 if (!match_attr)
3251                         return -EMSGSIZE;
3252                 err = devlink_dpipe_match_value_put(skb, &values[i]);
3253                 if (err)
3254                         goto err_match_value_put;
3255                 nla_nest_end(skb, match_attr);
3256         }
3257         return 0;
3258
3259 err_match_value_put:
3260         nla_nest_cancel(skb, match_attr);
3261         return err;
3262 }
3263
3264 static int devlink_dpipe_entry_put(struct sk_buff *skb,
3265                                    struct devlink_dpipe_entry *entry)
3266 {
3267         struct nlattr *entry_attr, *matches_attr, *actions_attr;
3268         int err;
3269
3270         entry_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_ENTRY);
3271         if (!entry_attr)
3272                 return  -EMSGSIZE;
3273
3274         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_INDEX, entry->index,
3275                               DEVLINK_ATTR_PAD))
3276                 goto nla_put_failure;
3277         if (entry->counter_valid)
3278                 if (nla_put_u64_64bit(skb, DEVLINK_ATTR_DPIPE_ENTRY_COUNTER,
3279                                       entry->counter, DEVLINK_ATTR_PAD))
3280                         goto nla_put_failure;
3281
3282         matches_attr = nla_nest_start_noflag(skb,
3283                                              DEVLINK_ATTR_DPIPE_ENTRY_MATCH_VALUES);
3284         if (!matches_attr)
3285                 goto nla_put_failure;
3286
3287         err = devlink_dpipe_match_values_put(skb, entry->match_values,
3288                                              entry->match_values_count);
3289         if (err) {
3290                 nla_nest_cancel(skb, matches_attr);
3291                 goto err_match_values_put;
3292         }
3293         nla_nest_end(skb, matches_attr);
3294
3295         actions_attr = nla_nest_start_noflag(skb,
3296                                              DEVLINK_ATTR_DPIPE_ENTRY_ACTION_VALUES);
3297         if (!actions_attr)
3298                 goto nla_put_failure;
3299
3300         err = devlink_dpipe_action_values_put(skb, entry->action_values,
3301                                               entry->action_values_count);
3302         if (err) {
3303                 nla_nest_cancel(skb, actions_attr);
3304                 goto err_action_values_put;
3305         }
3306         nla_nest_end(skb, actions_attr);
3307
3308         nla_nest_end(skb, entry_attr);
3309         return 0;
3310
3311 nla_put_failure:
3312         err = -EMSGSIZE;
3313 err_match_values_put:
3314 err_action_values_put:
3315         nla_nest_cancel(skb, entry_attr);
3316         return err;
3317 }
3318
3319 static struct devlink_dpipe_table *
3320 devlink_dpipe_table_find(struct list_head *dpipe_tables,
3321                          const char *table_name, struct devlink *devlink)
3322 {
3323         struct devlink_dpipe_table *table;
3324         list_for_each_entry_rcu(table, dpipe_tables, list,
3325                                 lockdep_is_held(&devlink->lock)) {
3326                 if (!strcmp(table->name, table_name))
3327                         return table;
3328         }
3329         return NULL;
3330 }
3331
3332 int devlink_dpipe_entry_ctx_prepare(struct devlink_dpipe_dump_ctx *dump_ctx)
3333 {
3334         struct devlink *devlink;
3335         int err;
3336
3337         err = devlink_dpipe_send_and_alloc_skb(&dump_ctx->skb,
3338                                                dump_ctx->info);
3339         if (err)
3340                 return err;
3341
3342         dump_ctx->hdr = genlmsg_put(dump_ctx->skb,
3343                                     dump_ctx->info->snd_portid,
3344                                     dump_ctx->info->snd_seq,
3345                                     &devlink_nl_family, NLM_F_MULTI,
3346                                     dump_ctx->cmd);
3347         if (!dump_ctx->hdr)
3348                 goto nla_put_failure;
3349
3350         devlink = dump_ctx->info->user_ptr[0];
3351         if (devlink_nl_put_handle(dump_ctx->skb, devlink))
3352                 goto nla_put_failure;
3353         dump_ctx->nest = nla_nest_start_noflag(dump_ctx->skb,
3354                                                DEVLINK_ATTR_DPIPE_ENTRIES);
3355         if (!dump_ctx->nest)
3356                 goto nla_put_failure;
3357         return 0;
3358
3359 nla_put_failure:
3360         nlmsg_free(dump_ctx->skb);
3361         return -EMSGSIZE;
3362 }
3363 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_prepare);
3364
3365 int devlink_dpipe_entry_ctx_append(struct devlink_dpipe_dump_ctx *dump_ctx,
3366                                    struct devlink_dpipe_entry *entry)
3367 {
3368         return devlink_dpipe_entry_put(dump_ctx->skb, entry);
3369 }
3370 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_append);
3371
3372 int devlink_dpipe_entry_ctx_close(struct devlink_dpipe_dump_ctx *dump_ctx)
3373 {
3374         nla_nest_end(dump_ctx->skb, dump_ctx->nest);
3375         genlmsg_end(dump_ctx->skb, dump_ctx->hdr);
3376         return 0;
3377 }
3378 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_ctx_close);
3379
3380 void devlink_dpipe_entry_clear(struct devlink_dpipe_entry *entry)
3381
3382 {
3383         unsigned int value_count, value_index;
3384         struct devlink_dpipe_value *value;
3385
3386         value = entry->action_values;
3387         value_count = entry->action_values_count;
3388         for (value_index = 0; value_index < value_count; value_index++) {
3389                 kfree(value[value_index].value);
3390                 kfree(value[value_index].mask);
3391         }
3392
3393         value = entry->match_values;
3394         value_count = entry->match_values_count;
3395         for (value_index = 0; value_index < value_count; value_index++) {
3396                 kfree(value[value_index].value);
3397                 kfree(value[value_index].mask);
3398         }
3399 }
3400 EXPORT_SYMBOL_GPL(devlink_dpipe_entry_clear);
3401
3402 static int devlink_dpipe_entries_fill(struct genl_info *info,
3403                                       enum devlink_command cmd, int flags,
3404                                       struct devlink_dpipe_table *table)
3405 {
3406         struct devlink_dpipe_dump_ctx dump_ctx;
3407         struct nlmsghdr *nlh;
3408         int err;
3409
3410         dump_ctx.skb = NULL;
3411         dump_ctx.cmd = cmd;
3412         dump_ctx.info = info;
3413
3414         err = table->table_ops->entries_dump(table->priv,
3415                                              table->counters_enabled,
3416                                              &dump_ctx);
3417         if (err)
3418                 return err;
3419
3420 send_done:
3421         nlh = nlmsg_put(dump_ctx.skb, info->snd_portid, info->snd_seq,
3422                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3423         if (!nlh) {
3424                 err = devlink_dpipe_send_and_alloc_skb(&dump_ctx.skb, info);
3425                 if (err)
3426                         return err;
3427                 goto send_done;
3428         }
3429         return genlmsg_reply(dump_ctx.skb, info);
3430 }
3431
3432 static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
3433                                             struct genl_info *info)
3434 {
3435         struct devlink *devlink = info->user_ptr[0];
3436         struct devlink_dpipe_table *table;
3437         const char *table_name;
3438
3439         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
3440                 return -EINVAL;
3441
3442         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3443         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3444                                          table_name, devlink);
3445         if (!table)
3446                 return -EINVAL;
3447
3448         if (!table->table_ops->entries_dump)
3449                 return -EINVAL;
3450
3451         return devlink_dpipe_entries_fill(info, DEVLINK_CMD_DPIPE_ENTRIES_GET,
3452                                           0, table);
3453 }
3454
3455 static int devlink_dpipe_fields_put(struct sk_buff *skb,
3456                                     const struct devlink_dpipe_header *header)
3457 {
3458         struct devlink_dpipe_field *field;
3459         struct nlattr *field_attr;
3460         int i;
3461
3462         for (i = 0; i < header->fields_count; i++) {
3463                 field = &header->fields[i];
3464                 field_attr = nla_nest_start_noflag(skb,
3465                                                    DEVLINK_ATTR_DPIPE_FIELD);
3466                 if (!field_attr)
3467                         return -EMSGSIZE;
3468                 if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_FIELD_NAME, field->name) ||
3469                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_ID, field->id) ||
3470                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_BITWIDTH, field->bitwidth) ||
3471                     nla_put_u32(skb, DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE, field->mapping_type))
3472                         goto nla_put_failure;
3473                 nla_nest_end(skb, field_attr);
3474         }
3475         return 0;
3476
3477 nla_put_failure:
3478         nla_nest_cancel(skb, field_attr);
3479         return -EMSGSIZE;
3480 }
3481
3482 static int devlink_dpipe_header_put(struct sk_buff *skb,
3483                                     struct devlink_dpipe_header *header)
3484 {
3485         struct nlattr *fields_attr, *header_attr;
3486         int err;
3487
3488         header_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADER);
3489         if (!header_attr)
3490                 return -EMSGSIZE;
3491
3492         if (nla_put_string(skb, DEVLINK_ATTR_DPIPE_HEADER_NAME, header->name) ||
3493             nla_put_u32(skb, DEVLINK_ATTR_DPIPE_HEADER_ID, header->id) ||
3494             nla_put_u8(skb, DEVLINK_ATTR_DPIPE_HEADER_GLOBAL, header->global))
3495                 goto nla_put_failure;
3496
3497         fields_attr = nla_nest_start_noflag(skb,
3498                                             DEVLINK_ATTR_DPIPE_HEADER_FIELDS);
3499         if (!fields_attr)
3500                 goto nla_put_failure;
3501
3502         err = devlink_dpipe_fields_put(skb, header);
3503         if (err) {
3504                 nla_nest_cancel(skb, fields_attr);
3505                 goto nla_put_failure;
3506         }
3507         nla_nest_end(skb, fields_attr);
3508         nla_nest_end(skb, header_attr);
3509         return 0;
3510
3511 nla_put_failure:
3512         err = -EMSGSIZE;
3513         nla_nest_cancel(skb, header_attr);
3514         return err;
3515 }
3516
3517 static int devlink_dpipe_headers_fill(struct genl_info *info,
3518                                       enum devlink_command cmd, int flags,
3519                                       struct devlink_dpipe_headers *
3520                                       dpipe_headers)
3521 {
3522         struct devlink *devlink = info->user_ptr[0];
3523         struct nlattr *headers_attr;
3524         struct sk_buff *skb = NULL;
3525         struct nlmsghdr *nlh;
3526         void *hdr;
3527         int i, j;
3528         int err;
3529
3530         i = 0;
3531 start_again:
3532         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3533         if (err)
3534                 return err;
3535
3536         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3537                           &devlink_nl_family, NLM_F_MULTI, cmd);
3538         if (!hdr) {
3539                 nlmsg_free(skb);
3540                 return -EMSGSIZE;
3541         }
3542
3543         if (devlink_nl_put_handle(skb, devlink))
3544                 goto nla_put_failure;
3545         headers_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_DPIPE_HEADERS);
3546         if (!headers_attr)
3547                 goto nla_put_failure;
3548
3549         j = 0;
3550         for (; i < dpipe_headers->headers_count; i++) {
3551                 err = devlink_dpipe_header_put(skb, dpipe_headers->headers[i]);
3552                 if (err) {
3553                         if (!j)
3554                                 goto err_table_put;
3555                         break;
3556                 }
3557                 j++;
3558         }
3559         nla_nest_end(skb, headers_attr);
3560         genlmsg_end(skb, hdr);
3561         if (i != dpipe_headers->headers_count)
3562                 goto start_again;
3563
3564 send_done:
3565         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3566                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3567         if (!nlh) {
3568                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3569                 if (err)
3570                         return err;
3571                 goto send_done;
3572         }
3573         return genlmsg_reply(skb, info);
3574
3575 nla_put_failure:
3576         err = -EMSGSIZE;
3577 err_table_put:
3578         nlmsg_free(skb);
3579         return err;
3580 }
3581
3582 static int devlink_nl_cmd_dpipe_headers_get(struct sk_buff *skb,
3583                                             struct genl_info *info)
3584 {
3585         struct devlink *devlink = info->user_ptr[0];
3586
3587         if (!devlink->dpipe_headers)
3588                 return -EOPNOTSUPP;
3589         return devlink_dpipe_headers_fill(info, DEVLINK_CMD_DPIPE_HEADERS_GET,
3590                                           0, devlink->dpipe_headers);
3591 }
3592
3593 static int devlink_dpipe_table_counters_set(struct devlink *devlink,
3594                                             const char *table_name,
3595                                             bool enable)
3596 {
3597         struct devlink_dpipe_table *table;
3598
3599         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
3600                                          table_name, devlink);
3601         if (!table)
3602                 return -EINVAL;
3603
3604         if (table->counter_control_extern)
3605                 return -EOPNOTSUPP;
3606
3607         if (!(table->counters_enabled ^ enable))
3608                 return 0;
3609
3610         table->counters_enabled = enable;
3611         if (table->table_ops->counters_set_update)
3612                 table->table_ops->counters_set_update(table->priv, enable);
3613         return 0;
3614 }
3615
3616 static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
3617                                                    struct genl_info *info)
3618 {
3619         struct devlink *devlink = info->user_ptr[0];
3620         const char *table_name;
3621         bool counters_enable;
3622
3623         if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
3624             !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
3625                 return -EINVAL;
3626
3627         table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
3628         counters_enable = !!nla_get_u8(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]);
3629
3630         return devlink_dpipe_table_counters_set(devlink, table_name,
3631                                                 counters_enable);
3632 }
3633
3634 static struct devlink_resource *
3635 devlink_resource_find(struct devlink *devlink,
3636                       struct devlink_resource *resource, u64 resource_id)
3637 {
3638         struct list_head *resource_list;
3639
3640         if (resource)
3641                 resource_list = &resource->resource_list;
3642         else
3643                 resource_list = &devlink->resource_list;
3644
3645         list_for_each_entry(resource, resource_list, list) {
3646                 struct devlink_resource *child_resource;
3647
3648                 if (resource->id == resource_id)
3649                         return resource;
3650
3651                 child_resource = devlink_resource_find(devlink, resource,
3652                                                        resource_id);
3653                 if (child_resource)
3654                         return child_resource;
3655         }
3656         return NULL;
3657 }
3658
3659 static void
3660 devlink_resource_validate_children(struct devlink_resource *resource)
3661 {
3662         struct devlink_resource *child_resource;
3663         bool size_valid = true;
3664         u64 parts_size = 0;
3665
3666         if (list_empty(&resource->resource_list))
3667                 goto out;
3668
3669         list_for_each_entry(child_resource, &resource->resource_list, list)
3670                 parts_size += child_resource->size_new;
3671
3672         if (parts_size > resource->size_new)
3673                 size_valid = false;
3674 out:
3675         resource->size_valid = size_valid;
3676 }
3677
3678 static int
3679 devlink_resource_validate_size(struct devlink_resource *resource, u64 size,
3680                                struct netlink_ext_ack *extack)
3681 {
3682         u64 reminder;
3683         int err = 0;
3684
3685         if (size > resource->size_params.size_max) {
3686                 NL_SET_ERR_MSG_MOD(extack, "Size larger than maximum");
3687                 err = -EINVAL;
3688         }
3689
3690         if (size < resource->size_params.size_min) {
3691                 NL_SET_ERR_MSG_MOD(extack, "Size smaller than minimum");
3692                 err = -EINVAL;
3693         }
3694
3695         div64_u64_rem(size, resource->size_params.size_granularity, &reminder);
3696         if (reminder) {
3697                 NL_SET_ERR_MSG_MOD(extack, "Wrong granularity");
3698                 err = -EINVAL;
3699         }
3700
3701         return err;
3702 }
3703
3704 static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
3705                                        struct genl_info *info)
3706 {
3707         struct devlink *devlink = info->user_ptr[0];
3708         struct devlink_resource *resource;
3709         u64 resource_id;
3710         u64 size;
3711         int err;
3712
3713         if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
3714             !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
3715                 return -EINVAL;
3716         resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
3717
3718         resource = devlink_resource_find(devlink, NULL, resource_id);
3719         if (!resource)
3720                 return -EINVAL;
3721
3722         size = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]);
3723         err = devlink_resource_validate_size(resource, size, info->extack);
3724         if (err)
3725                 return err;
3726
3727         resource->size_new = size;
3728         devlink_resource_validate_children(resource);
3729         if (resource->parent)
3730                 devlink_resource_validate_children(resource->parent);
3731         return 0;
3732 }
3733
3734 static int
3735 devlink_resource_size_params_put(struct devlink_resource *resource,
3736                                  struct sk_buff *skb)
3737 {
3738         struct devlink_resource_size_params *size_params;
3739
3740         size_params = &resource->size_params;
3741         if (nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_GRAN,
3742                               size_params->size_granularity, DEVLINK_ATTR_PAD) ||
3743             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MAX,
3744                               size_params->size_max, DEVLINK_ATTR_PAD) ||
3745             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_MIN,
3746                               size_params->size_min, DEVLINK_ATTR_PAD) ||
3747             nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_UNIT, size_params->unit))
3748                 return -EMSGSIZE;
3749         return 0;
3750 }
3751
3752 static int devlink_resource_occ_put(struct devlink_resource *resource,
3753                                     struct sk_buff *skb)
3754 {
3755         if (!resource->occ_get)
3756                 return 0;
3757         return nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_OCC,
3758                                  resource->occ_get(resource->occ_get_priv),
3759                                  DEVLINK_ATTR_PAD);
3760 }
3761
3762 static int devlink_resource_put(struct devlink *devlink, struct sk_buff *skb,
3763                                 struct devlink_resource *resource)
3764 {
3765         struct devlink_resource *child_resource;
3766         struct nlattr *child_resource_attr;
3767         struct nlattr *resource_attr;
3768
3769         resource_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_RESOURCE);
3770         if (!resource_attr)
3771                 return -EMSGSIZE;
3772
3773         if (nla_put_string(skb, DEVLINK_ATTR_RESOURCE_NAME, resource->name) ||
3774             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE, resource->size,
3775                               DEVLINK_ATTR_PAD) ||
3776             nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_ID, resource->id,
3777                               DEVLINK_ATTR_PAD))
3778                 goto nla_put_failure;
3779         if (resource->size != resource->size_new)
3780                 nla_put_u64_64bit(skb, DEVLINK_ATTR_RESOURCE_SIZE_NEW,
3781                                   resource->size_new, DEVLINK_ATTR_PAD);
3782         if (devlink_resource_occ_put(resource, skb))
3783                 goto nla_put_failure;
3784         if (devlink_resource_size_params_put(resource, skb))
3785                 goto nla_put_failure;
3786         if (list_empty(&resource->resource_list))
3787                 goto out;
3788
3789         if (nla_put_u8(skb, DEVLINK_ATTR_RESOURCE_SIZE_VALID,
3790                        resource->size_valid))
3791                 goto nla_put_failure;
3792
3793         child_resource_attr = nla_nest_start_noflag(skb,
3794                                                     DEVLINK_ATTR_RESOURCE_LIST);
3795         if (!child_resource_attr)
3796                 goto nla_put_failure;
3797
3798         list_for_each_entry(child_resource, &resource->resource_list, list) {
3799                 if (devlink_resource_put(devlink, skb, child_resource))
3800                         goto resource_put_failure;
3801         }
3802
3803         nla_nest_end(skb, child_resource_attr);
3804 out:
3805         nla_nest_end(skb, resource_attr);
3806         return 0;
3807
3808 resource_put_failure:
3809         nla_nest_cancel(skb, child_resource_attr);
3810 nla_put_failure:
3811         nla_nest_cancel(skb, resource_attr);
3812         return -EMSGSIZE;
3813 }
3814
3815 static int devlink_resource_fill(struct genl_info *info,
3816                                  enum devlink_command cmd, int flags)
3817 {
3818         struct devlink *devlink = info->user_ptr[0];
3819         struct devlink_resource *resource;
3820         struct nlattr *resources_attr;
3821         struct sk_buff *skb = NULL;
3822         struct nlmsghdr *nlh;
3823         bool incomplete;
3824         void *hdr;
3825         int i;
3826         int err;
3827
3828         resource = list_first_entry(&devlink->resource_list,
3829                                     struct devlink_resource, list);
3830 start_again:
3831         err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3832         if (err)
3833                 return err;
3834
3835         hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
3836                           &devlink_nl_family, NLM_F_MULTI, cmd);
3837         if (!hdr) {
3838                 nlmsg_free(skb);
3839                 return -EMSGSIZE;
3840         }
3841
3842         if (devlink_nl_put_handle(skb, devlink))
3843                 goto nla_put_failure;
3844
3845         resources_attr = nla_nest_start_noflag(skb,
3846                                                DEVLINK_ATTR_RESOURCE_LIST);
3847         if (!resources_attr)
3848                 goto nla_put_failure;
3849
3850         incomplete = false;
3851         i = 0;
3852         list_for_each_entry_from(resource, &devlink->resource_list, list) {
3853                 err = devlink_resource_put(devlink, skb, resource);
3854                 if (err) {
3855                         if (!i)
3856                                 goto err_resource_put;
3857                         incomplete = true;
3858                         break;
3859                 }
3860                 i++;
3861         }
3862         nla_nest_end(skb, resources_attr);
3863         genlmsg_end(skb, hdr);
3864         if (incomplete)
3865                 goto start_again;
3866 send_done:
3867         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
3868                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
3869         if (!nlh) {
3870                 err = devlink_dpipe_send_and_alloc_skb(&skb, info);
3871                 if (err)
3872                         return err;
3873                 goto send_done;
3874         }
3875         return genlmsg_reply(skb, info);
3876
3877 nla_put_failure:
3878         err = -EMSGSIZE;
3879 err_resource_put:
3880         nlmsg_free(skb);
3881         return err;
3882 }
3883
3884 static int devlink_nl_cmd_resource_dump(struct sk_buff *skb,
3885                                         struct genl_info *info)
3886 {
3887         struct devlink *devlink = info->user_ptr[0];
3888
3889         if (list_empty(&devlink->resource_list))
3890                 return -EOPNOTSUPP;
3891
3892         return devlink_resource_fill(info, DEVLINK_CMD_RESOURCE_DUMP, 0);
3893 }
3894
3895 static int
3896 devlink_resources_validate(struct devlink *devlink,
3897                            struct devlink_resource *resource,
3898                            struct genl_info *info)
3899 {
3900         struct list_head *resource_list;
3901         int err = 0;
3902
3903         if (resource)
3904                 resource_list = &resource->resource_list;
3905         else
3906                 resource_list = &devlink->resource_list;
3907
3908         list_for_each_entry(resource, resource_list, list) {
3909                 if (!resource->size_valid)
3910                         return -EINVAL;
3911                 err = devlink_resources_validate(devlink, resource, info);
3912                 if (err)
3913                         return err;
3914         }
3915         return err;
3916 }
3917
3918 static struct net *devlink_netns_get(struct sk_buff *skb,
3919                                      struct genl_info *info)
3920 {
3921         struct nlattr *netns_pid_attr = info->attrs[DEVLINK_ATTR_NETNS_PID];
3922         struct nlattr *netns_fd_attr = info->attrs[DEVLINK_ATTR_NETNS_FD];
3923         struct nlattr *netns_id_attr = info->attrs[DEVLINK_ATTR_NETNS_ID];
3924         struct net *net;
3925
3926         if (!!netns_pid_attr + !!netns_fd_attr + !!netns_id_attr > 1) {
3927                 NL_SET_ERR_MSG_MOD(info->extack, "multiple netns identifying attributes specified");
3928                 return ERR_PTR(-EINVAL);
3929         }
3930
3931         if (netns_pid_attr) {
3932                 net = get_net_ns_by_pid(nla_get_u32(netns_pid_attr));
3933         } else if (netns_fd_attr) {
3934                 net = get_net_ns_by_fd(nla_get_u32(netns_fd_attr));
3935         } else if (netns_id_attr) {
3936                 net = get_net_ns_by_id(sock_net(skb->sk),
3937                                        nla_get_u32(netns_id_attr));
3938                 if (!net)
3939                         net = ERR_PTR(-EINVAL);
3940         } else {
3941                 WARN_ON(1);
3942                 net = ERR_PTR(-EINVAL);
3943         }
3944         if (IS_ERR(net)) {
3945                 NL_SET_ERR_MSG_MOD(info->extack, "Unknown network namespace");
3946                 return ERR_PTR(-EINVAL);
3947         }
3948         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
3949                 put_net(net);
3950                 return ERR_PTR(-EPERM);
3951         }
3952         return net;
3953 }
3954
3955 static void devlink_param_notify(struct devlink *devlink,
3956                                  unsigned int port_index,
3957                                  struct devlink_param_item *param_item,
3958                                  enum devlink_command cmd);
3959
3960 static void devlink_ns_change_notify(struct devlink *devlink,
3961                                      struct net *dest_net, struct net *curr_net,
3962                                      bool new)
3963 {
3964         struct devlink_param_item *param_item;
3965         enum devlink_command cmd;
3966
3967         /* Userspace needs to be notified about devlink objects
3968          * removed from original and entering new network namespace.
3969          * The rest of the devlink objects are re-created during
3970          * reload process so the notifications are generated separatelly.
3971          */
3972
3973         if (!dest_net || net_eq(dest_net, curr_net))
3974                 return;
3975
3976         if (new)
3977                 devlink_notify(devlink, DEVLINK_CMD_NEW);
3978
3979         cmd = new ? DEVLINK_CMD_PARAM_NEW : DEVLINK_CMD_PARAM_DEL;
3980         list_for_each_entry(param_item, &devlink->param_list, list)
3981                 devlink_param_notify(devlink, 0, param_item, cmd);
3982
3983         if (!new)
3984                 devlink_notify(devlink, DEVLINK_CMD_DEL);
3985 }
3986
3987 static bool devlink_reload_supported(const struct devlink_ops *ops)
3988 {
3989         return ops->reload_down && ops->reload_up;
3990 }
3991
3992 static void devlink_reload_failed_set(struct devlink *devlink,
3993                                       bool reload_failed)
3994 {
3995         if (devlink->reload_failed == reload_failed)
3996                 return;
3997         devlink->reload_failed = reload_failed;
3998         devlink_notify(devlink, DEVLINK_CMD_NEW);
3999 }
4000
4001 bool devlink_is_reload_failed(const struct devlink *devlink)
4002 {
4003         return devlink->reload_failed;
4004 }
4005 EXPORT_SYMBOL_GPL(devlink_is_reload_failed);
4006
4007 static void
4008 __devlink_reload_stats_update(struct devlink *devlink, u32 *reload_stats,
4009                               enum devlink_reload_limit limit, u32 actions_performed)
4010 {
4011         unsigned long actions = actions_performed;
4012         int stat_idx;
4013         int action;
4014
4015         for_each_set_bit(action, &actions, __DEVLINK_RELOAD_ACTION_MAX) {
4016                 stat_idx = limit * __DEVLINK_RELOAD_ACTION_MAX + action;
4017                 reload_stats[stat_idx]++;
4018         }
4019         devlink_notify(devlink, DEVLINK_CMD_NEW);
4020 }
4021
4022 static void
4023 devlink_reload_stats_update(struct devlink *devlink, enum devlink_reload_limit limit,
4024                             u32 actions_performed)
4025 {
4026         __devlink_reload_stats_update(devlink, devlink->stats.reload_stats, limit,
4027                                       actions_performed);
4028 }
4029
4030 /**
4031  *      devlink_remote_reload_actions_performed - Update devlink on reload actions
4032  *        performed which are not a direct result of devlink reload call.
4033  *
4034  *      This should be called by a driver after performing reload actions in case it was not
4035  *      a result of devlink reload call. For example fw_activate was performed as a result
4036  *      of devlink reload triggered fw_activate on another host.
4037  *      The motivation for this function is to keep data on reload actions performed on this
4038  *      function whether it was done due to direct devlink reload call or not.
4039  *
4040  *      @devlink: devlink
4041  *      @limit: reload limit
4042  *      @actions_performed: bitmask of actions performed
4043  */
4044 void devlink_remote_reload_actions_performed(struct devlink *devlink,
4045                                              enum devlink_reload_limit limit,
4046                                              u32 actions_performed)
4047 {
4048         if (WARN_ON(!actions_performed ||
4049                     actions_performed & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
4050                     actions_performed >= BIT(__DEVLINK_RELOAD_ACTION_MAX) ||
4051                     limit > DEVLINK_RELOAD_LIMIT_MAX))
4052                 return;
4053
4054         __devlink_reload_stats_update(devlink, devlink->stats.remote_reload_stats, limit,
4055                                       actions_performed);
4056 }
4057 EXPORT_SYMBOL_GPL(devlink_remote_reload_actions_performed);
4058
4059 static int devlink_reload(struct devlink *devlink, struct net *dest_net,
4060                           enum devlink_reload_action action, enum devlink_reload_limit limit,
4061                           u32 *actions_performed, struct netlink_ext_ack *extack)
4062 {
4063         u32 remote_reload_stats[DEVLINK_RELOAD_STATS_ARRAY_SIZE];
4064         struct net *curr_net;
4065         int err;
4066
4067         memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
4068                sizeof(remote_reload_stats));
4069
4070         curr_net = devlink_net(devlink);
4071         devlink_ns_change_notify(devlink, dest_net, curr_net, false);
4072         err = devlink->ops->reload_down(devlink, !!dest_net, action, limit, extack);
4073         if (err)
4074                 return err;
4075
4076         if (dest_net && !net_eq(dest_net, curr_net))
4077                 write_pnet(&devlink->_net, dest_net);
4078
4079         err = devlink->ops->reload_up(devlink, action, limit, actions_performed, extack);
4080         devlink_reload_failed_set(devlink, !!err);
4081         if (err)
4082                 return err;
4083
4084         devlink_ns_change_notify(devlink, dest_net, curr_net, true);
4085         WARN_ON(!(*actions_performed & BIT(action)));
4086         /* Catch driver on updating the remote action within devlink reload */
4087         WARN_ON(memcmp(remote_reload_stats, devlink->stats.remote_reload_stats,
4088                        sizeof(remote_reload_stats)));
4089         devlink_reload_stats_update(devlink, limit, *actions_performed);
4090         return 0;
4091 }
4092
4093 static int
4094 devlink_nl_reload_actions_performed_snd(struct devlink *devlink, u32 actions_performed,
4095                                         enum devlink_command cmd, struct genl_info *info)
4096 {
4097         struct sk_buff *msg;
4098         void *hdr;
4099
4100         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4101         if (!msg)
4102                 return -ENOMEM;
4103
4104         hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, &devlink_nl_family, 0, cmd);
4105         if (!hdr)
4106                 goto free_msg;
4107
4108         if (devlink_nl_put_handle(msg, devlink))
4109                 goto nla_put_failure;
4110
4111         if (nla_put_bitfield32(msg, DEVLINK_ATTR_RELOAD_ACTIONS_PERFORMED, actions_performed,
4112                                actions_performed))
4113                 goto nla_put_failure;
4114         genlmsg_end(msg, hdr);
4115
4116         return genlmsg_reply(msg, info);
4117
4118 nla_put_failure:
4119         genlmsg_cancel(msg, hdr);
4120 free_msg:
4121         nlmsg_free(msg);
4122         return -EMSGSIZE;
4123 }
4124
4125 static int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info)
4126 {
4127         struct devlink *devlink = info->user_ptr[0];
4128         enum devlink_reload_action action;
4129         enum devlink_reload_limit limit;
4130         struct net *dest_net = NULL;
4131         u32 actions_performed;
4132         int err;
4133
4134         if (!(devlink->features & DEVLINK_F_RELOAD))
4135                 return -EOPNOTSUPP;
4136
4137         err = devlink_resources_validate(devlink, NULL, info);
4138         if (err) {
4139                 NL_SET_ERR_MSG_MOD(info->extack, "resources size validation failed");
4140                 return err;
4141         }
4142
4143         if (info->attrs[DEVLINK_ATTR_RELOAD_ACTION])
4144                 action = nla_get_u8(info->attrs[DEVLINK_ATTR_RELOAD_ACTION]);
4145         else
4146                 action = DEVLINK_RELOAD_ACTION_DRIVER_REINIT;
4147
4148         if (!devlink_reload_action_is_supported(devlink, action)) {
4149                 NL_SET_ERR_MSG_MOD(info->extack,
4150                                    "Requested reload action is not supported by the driver");
4151                 return -EOPNOTSUPP;
4152         }
4153
4154         limit = DEVLINK_RELOAD_LIMIT_UNSPEC;
4155         if (info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]) {
4156                 struct nla_bitfield32 limits;
4157                 u32 limits_selected;
4158
4159                 limits = nla_get_bitfield32(info->attrs[DEVLINK_ATTR_RELOAD_LIMITS]);
4160                 limits_selected = limits.value & limits.selector;
4161                 if (!limits_selected) {
4162                         NL_SET_ERR_MSG_MOD(info->extack, "Invalid limit selected");
4163                         return -EINVAL;
4164                 }
4165                 for (limit = 0 ; limit <= DEVLINK_RELOAD_LIMIT_MAX ; limit++)
4166                         if (limits_selected & BIT(limit))
4167                                 break;
4168                 /* UAPI enables multiselection, but currently it is not used */
4169                 if (limits_selected != BIT(limit)) {
4170                         NL_SET_ERR_MSG_MOD(info->extack,
4171                                            "Multiselection of limit is not supported");
4172                         return -EOPNOTSUPP;
4173                 }
4174                 if (!devlink_reload_limit_is_supported(devlink, limit)) {
4175                         NL_SET_ERR_MSG_MOD(info->extack,
4176                                            "Requested limit is not supported by the driver");
4177                         return -EOPNOTSUPP;
4178                 }
4179                 if (devlink_reload_combination_is_invalid(action, limit)) {
4180                         NL_SET_ERR_MSG_MOD(info->extack,
4181                                            "Requested limit is invalid for this action");
4182                         return -EINVAL;
4183                 }
4184         }
4185         if (info->attrs[DEVLINK_ATTR_NETNS_PID] ||
4186             info->attrs[DEVLINK_ATTR_NETNS_FD] ||
4187             info->attrs[DEVLINK_ATTR_NETNS_ID]) {
4188                 dest_net = devlink_netns_get(skb, info);
4189                 if (IS_ERR(dest_net))
4190                         return PTR_ERR(dest_net);
4191         }
4192
4193         err = devlink_reload(devlink, dest_net, action, limit, &actions_performed, info->extack);
4194
4195         if (dest_net)
4196                 put_net(dest_net);
4197
4198         if (err)
4199                 return err;
4200         /* For backward compatibility generate reply only if attributes used by user */
4201         if (!info->attrs[DEVLINK_ATTR_RELOAD_ACTION] && !info->attrs[DEVLINK_ATTR_RELOAD_LIMITS])
4202                 return 0;
4203
4204         return devlink_nl_reload_actions_performed_snd(devlink, actions_performed,
4205                                                        DEVLINK_CMD_RELOAD, info);
4206 }
4207
4208 static int devlink_nl_flash_update_fill(struct sk_buff *msg,
4209                                         struct devlink *devlink,
4210                                         enum devlink_command cmd,
4211                                         struct devlink_flash_notify *params)
4212 {
4213         void *hdr;
4214
4215         hdr = genlmsg_put(msg, 0, 0, &devlink_nl_family, 0, cmd);
4216         if (!hdr)
4217                 return -EMSGSIZE;
4218
4219         if (devlink_nl_put_handle(msg, devlink))
4220                 goto nla_put_failure;
4221
4222         if (cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS)
4223                 goto out;
4224
4225         if (params->status_msg &&
4226             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_MSG,
4227                            params->status_msg))
4228                 goto nla_put_failure;
4229         if (params->component &&
4230             nla_put_string(msg, DEVLINK_ATTR_FLASH_UPDATE_COMPONENT,
4231                            params->component))
4232                 goto nla_put_failure;
4233         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_DONE,
4234                               params->done, DEVLINK_ATTR_PAD))
4235                 goto nla_put_failure;
4236         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TOTAL,
4237                               params->total, DEVLINK_ATTR_PAD))
4238                 goto nla_put_failure;
4239         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_FLASH_UPDATE_STATUS_TIMEOUT,
4240                               params->timeout, DEVLINK_ATTR_PAD))
4241                 goto nla_put_failure;
4242
4243 out:
4244         genlmsg_end(msg, hdr);
4245         return 0;
4246
4247 nla_put_failure:
4248         genlmsg_cancel(msg, hdr);
4249         return -EMSGSIZE;
4250 }
4251
4252 static void __devlink_flash_update_notify(struct devlink *devlink,
4253                                           enum devlink_command cmd,
4254                                           struct devlink_flash_notify *params)
4255 {
4256         struct sk_buff *msg;
4257         int err;
4258
4259         WARN_ON(cmd != DEVLINK_CMD_FLASH_UPDATE &&
4260                 cmd != DEVLINK_CMD_FLASH_UPDATE_END &&
4261                 cmd != DEVLINK_CMD_FLASH_UPDATE_STATUS);
4262
4263         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
4264                 return;
4265
4266         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4267         if (!msg)
4268                 return;
4269
4270         err = devlink_nl_flash_update_fill(msg, devlink, cmd, params);
4271         if (err)
4272                 goto out_free_msg;
4273
4274         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4275                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4276         return;
4277
4278 out_free_msg:
4279         nlmsg_free(msg);
4280 }
4281
4282 static void devlink_flash_update_begin_notify(struct devlink *devlink)
4283 {
4284         struct devlink_flash_notify params = {};
4285
4286         __devlink_flash_update_notify(devlink,
4287                                       DEVLINK_CMD_FLASH_UPDATE,
4288                                       &params);
4289 }
4290
4291 static void devlink_flash_update_end_notify(struct devlink *devlink)
4292 {
4293         struct devlink_flash_notify params = {};
4294
4295         __devlink_flash_update_notify(devlink,
4296                                       DEVLINK_CMD_FLASH_UPDATE_END,
4297                                       &params);
4298 }
4299
4300 void devlink_flash_update_status_notify(struct devlink *devlink,
4301                                         const char *status_msg,
4302                                         const char *component,
4303                                         unsigned long done,
4304                                         unsigned long total)
4305 {
4306         struct devlink_flash_notify params = {
4307                 .status_msg = status_msg,
4308                 .component = component,
4309                 .done = done,
4310                 .total = total,
4311         };
4312
4313         __devlink_flash_update_notify(devlink,
4314                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4315                                       &params);
4316 }
4317 EXPORT_SYMBOL_GPL(devlink_flash_update_status_notify);
4318
4319 void devlink_flash_update_timeout_notify(struct devlink *devlink,
4320                                          const char *status_msg,
4321                                          const char *component,
4322                                          unsigned long timeout)
4323 {
4324         struct devlink_flash_notify params = {
4325                 .status_msg = status_msg,
4326                 .component = component,
4327                 .timeout = timeout,
4328         };
4329
4330         __devlink_flash_update_notify(devlink,
4331                                       DEVLINK_CMD_FLASH_UPDATE_STATUS,
4332                                       &params);
4333 }
4334 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
4335
4336 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
4337                                        struct genl_info *info)
4338 {
4339         struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name;
4340         struct devlink_flash_update_params params = {};
4341         struct devlink *devlink = info->user_ptr[0];
4342         const char *file_name;
4343         u32 supported_params;
4344         int ret;
4345
4346         if (!devlink->ops->flash_update)
4347                 return -EOPNOTSUPP;
4348
4349         if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
4350                 return -EINVAL;
4351
4352         supported_params = devlink->ops->supported_flash_update_params;
4353
4354         nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
4355         if (nla_component) {
4356                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
4357                         NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
4358                                             "component update is not supported by this device");
4359                         return -EOPNOTSUPP;
4360                 }
4361                 params.component = nla_data(nla_component);
4362         }
4363
4364         nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
4365         if (nla_overwrite_mask) {
4366                 struct nla_bitfield32 sections;
4367
4368                 if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK)) {
4369                         NL_SET_ERR_MSG_ATTR(info->extack, nla_overwrite_mask,
4370                                             "overwrite settings are not supported by this device");
4371                         return -EOPNOTSUPP;
4372                 }
4373                 sections = nla_get_bitfield32(nla_overwrite_mask);
4374                 params.overwrite_mask = sections.value & sections.selector;
4375         }
4376
4377         nla_file_name = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME];
4378         file_name = nla_data(nla_file_name);
4379         ret = request_firmware(&params.fw, file_name, devlink->dev);
4380         if (ret) {
4381                 NL_SET_ERR_MSG_ATTR(info->extack, nla_file_name, "failed to locate the requested firmware file");
4382                 return ret;
4383         }
4384
4385         devlink_flash_update_begin_notify(devlink);
4386         ret = devlink->ops->flash_update(devlink, &params, info->extack);
4387         devlink_flash_update_end_notify(devlink);
4388
4389         release_firmware(params.fw);
4390
4391         return ret;
4392 }
4393
4394 static const struct devlink_param devlink_param_generic[] = {
4395         {
4396                 .id = DEVLINK_PARAM_GENERIC_ID_INT_ERR_RESET,
4397                 .name = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_NAME,
4398                 .type = DEVLINK_PARAM_GENERIC_INT_ERR_RESET_TYPE,
4399         },
4400         {
4401                 .id = DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
4402                 .name = DEVLINK_PARAM_GENERIC_MAX_MACS_NAME,
4403                 .type = DEVLINK_PARAM_GENERIC_MAX_MACS_TYPE,
4404         },
4405         {
4406                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SRIOV,
4407                 .name = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_NAME,
4408                 .type = DEVLINK_PARAM_GENERIC_ENABLE_SRIOV_TYPE,
4409         },
4410         {
4411                 .id = DEVLINK_PARAM_GENERIC_ID_REGION_SNAPSHOT,
4412                 .name = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_NAME,
4413                 .type = DEVLINK_PARAM_GENERIC_REGION_SNAPSHOT_TYPE,
4414         },
4415         {
4416                 .id = DEVLINK_PARAM_GENERIC_ID_IGNORE_ARI,
4417                 .name = DEVLINK_PARAM_GENERIC_IGNORE_ARI_NAME,
4418                 .type = DEVLINK_PARAM_GENERIC_IGNORE_ARI_TYPE,
4419         },
4420         {
4421                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MAX,
4422                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_NAME,
4423                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MAX_TYPE,
4424         },
4425         {
4426                 .id = DEVLINK_PARAM_GENERIC_ID_MSIX_VEC_PER_PF_MIN,
4427                 .name = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_NAME,
4428                 .type = DEVLINK_PARAM_GENERIC_MSIX_VEC_PER_PF_MIN_TYPE,
4429         },
4430         {
4431                 .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,
4432                 .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,
4433                 .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,
4434         },
4435         {
4436                 .id = DEVLINK_PARAM_GENERIC_ID_RESET_DEV_ON_DRV_PROBE,
4437                 .name = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_NAME,
4438                 .type = DEVLINK_PARAM_GENERIC_RESET_DEV_ON_DRV_PROBE_TYPE,
4439         },
4440         {
4441                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
4442                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_NAME,
4443                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ROCE_TYPE,
4444         },
4445         {
4446                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_REMOTE_DEV_RESET,
4447                 .name = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_NAME,
4448                 .type = DEVLINK_PARAM_GENERIC_ENABLE_REMOTE_DEV_RESET_TYPE,
4449         },
4450         {
4451                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
4452                 .name = DEVLINK_PARAM_GENERIC_ENABLE_ETH_NAME,
4453                 .type = DEVLINK_PARAM_GENERIC_ENABLE_ETH_TYPE,
4454         },
4455         {
4456                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
4457                 .name = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_NAME,
4458                 .type = DEVLINK_PARAM_GENERIC_ENABLE_RDMA_TYPE,
4459         },
4460         {
4461                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
4462                 .name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME,
4463                 .type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE,
4464         },
4465         {
4466                 .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
4467                 .name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME,
4468                 .type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE,
4469         },
4470         {
4471                 .id = DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
4472                 .name = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_NAME,
4473                 .type = DEVLINK_PARAM_GENERIC_IO_EQ_SIZE_TYPE,
4474         },
4475         {
4476                 .id = DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
4477                 .name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
4478                 .type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
4479         },
4480 };
4481
4482 static int devlink_param_generic_verify(const struct devlink_param *param)
4483 {
4484         /* verify it match generic parameter by id and name */
4485         if (param->id > DEVLINK_PARAM_GENERIC_ID_MAX)
4486                 return -EINVAL;
4487         if (strcmp(param->name, devlink_param_generic[param->id].name))
4488                 return -ENOENT;
4489
4490         WARN_ON(param->type != devlink_param_generic[param->id].type);
4491
4492         return 0;
4493 }
4494
4495 static int devlink_param_driver_verify(const struct devlink_param *param)
4496 {
4497         int i;
4498
4499         if (param->id <= DEVLINK_PARAM_GENERIC_ID_MAX)
4500                 return -EINVAL;
4501         /* verify no such name in generic params */
4502         for (i = 0; i <= DEVLINK_PARAM_GENERIC_ID_MAX; i++)
4503                 if (!strcmp(param->name, devlink_param_generic[i].name))
4504                         return -EEXIST;
4505
4506         return 0;
4507 }
4508
4509 static struct devlink_param_item *
4510 devlink_param_find_by_name(struct list_head *param_list,
4511                            const char *param_name)
4512 {
4513         struct devlink_param_item *param_item;
4514
4515         list_for_each_entry(param_item, param_list, list)
4516                 if (!strcmp(param_item->param->name, param_name))
4517                         return param_item;
4518         return NULL;
4519 }
4520
4521 static struct devlink_param_item *
4522 devlink_param_find_by_id(struct list_head *param_list, u32 param_id)
4523 {
4524         struct devlink_param_item *param_item;
4525
4526         list_for_each_entry(param_item, param_list, list)
4527                 if (param_item->param->id == param_id)
4528                         return param_item;
4529         return NULL;
4530 }
4531
4532 static bool
4533 devlink_param_cmode_is_supported(const struct devlink_param *param,
4534                                  enum devlink_param_cmode cmode)
4535 {
4536         return test_bit(cmode, &param->supported_cmodes);
4537 }
4538
4539 static int devlink_param_get(struct devlink *devlink,
4540                              const struct devlink_param *param,
4541                              struct devlink_param_gset_ctx *ctx)
4542 {
4543         if (!param->get)
4544                 return -EOPNOTSUPP;
4545         return param->get(devlink, param->id, ctx);
4546 }
4547
4548 static int devlink_param_set(struct devlink *devlink,
4549                              const struct devlink_param *param,
4550                              struct devlink_param_gset_ctx *ctx)
4551 {
4552         if (!param->set)
4553                 return -EOPNOTSUPP;
4554         return param->set(devlink, param->id, ctx);
4555 }
4556
4557 static int
4558 devlink_param_type_to_nla_type(enum devlink_param_type param_type)
4559 {
4560         switch (param_type) {
4561         case DEVLINK_PARAM_TYPE_U8:
4562                 return NLA_U8;
4563         case DEVLINK_PARAM_TYPE_U16:
4564                 return NLA_U16;
4565         case DEVLINK_PARAM_TYPE_U32:
4566                 return NLA_U32;
4567         case DEVLINK_PARAM_TYPE_STRING:
4568                 return NLA_STRING;
4569         case DEVLINK_PARAM_TYPE_BOOL:
4570                 return NLA_FLAG;
4571         default:
4572                 return -EINVAL;
4573         }
4574 }
4575
4576 static int
4577 devlink_nl_param_value_fill_one(struct sk_buff *msg,
4578                                 enum devlink_param_type type,
4579                                 enum devlink_param_cmode cmode,
4580                                 union devlink_param_value val)
4581 {
4582         struct nlattr *param_value_attr;
4583
4584         param_value_attr = nla_nest_start_noflag(msg,
4585                                                  DEVLINK_ATTR_PARAM_VALUE);
4586         if (!param_value_attr)
4587                 goto nla_put_failure;
4588
4589         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_CMODE, cmode))
4590                 goto value_nest_cancel;
4591
4592         switch (type) {
4593         case DEVLINK_PARAM_TYPE_U8:
4594                 if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu8))
4595                         goto value_nest_cancel;
4596                 break;
4597         case DEVLINK_PARAM_TYPE_U16:
4598                 if (nla_put_u16(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu16))
4599                         goto value_nest_cancel;
4600                 break;
4601         case DEVLINK_PARAM_TYPE_U32:
4602                 if (nla_put_u32(msg, DEVLINK_ATTR_PARAM_VALUE_DATA, val.vu32))
4603                         goto value_nest_cancel;
4604                 break;
4605         case DEVLINK_PARAM_TYPE_STRING:
4606                 if (nla_put_string(msg, DEVLINK_ATTR_PARAM_VALUE_DATA,
4607                                    val.vstr))
4608                         goto value_nest_cancel;
4609                 break;
4610         case DEVLINK_PARAM_TYPE_BOOL:
4611                 if (val.vbool &&
4612                     nla_put_flag(msg, DEVLINK_ATTR_PARAM_VALUE_DATA))
4613                         goto value_nest_cancel;
4614                 break;
4615         }
4616
4617         nla_nest_end(msg, param_value_attr);
4618         return 0;
4619
4620 value_nest_cancel:
4621         nla_nest_cancel(msg, param_value_attr);
4622 nla_put_failure:
4623         return -EMSGSIZE;
4624 }
4625
4626 static int devlink_nl_param_fill(struct sk_buff *msg, struct devlink *devlink,
4627                                  unsigned int port_index,
4628                                  struct devlink_param_item *param_item,
4629                                  enum devlink_command cmd,
4630                                  u32 portid, u32 seq, int flags)
4631 {
4632         union devlink_param_value param_value[DEVLINK_PARAM_CMODE_MAX + 1];
4633         bool param_value_set[DEVLINK_PARAM_CMODE_MAX + 1] = {};
4634         const struct devlink_param *param = param_item->param;
4635         struct devlink_param_gset_ctx ctx;
4636         struct nlattr *param_values_list;
4637         struct nlattr *param_attr;
4638         int nla_type;
4639         void *hdr;
4640         int err;
4641         int i;
4642
4643         /* Get value from driver part to driverinit configuration mode */
4644         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
4645                 if (!devlink_param_cmode_is_supported(param, i))
4646                         continue;
4647                 if (i == DEVLINK_PARAM_CMODE_DRIVERINIT) {
4648                         if (!param_item->driverinit_value_valid)
4649                                 return -EOPNOTSUPP;
4650                         param_value[i] = param_item->driverinit_value;
4651                 } else {
4652                         ctx.cmode = i;
4653                         err = devlink_param_get(devlink, param, &ctx);
4654                         if (err)
4655                                 return err;
4656                         param_value[i] = ctx.val;
4657                 }
4658                 param_value_set[i] = true;
4659         }
4660
4661         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
4662         if (!hdr)
4663                 return -EMSGSIZE;
4664
4665         if (devlink_nl_put_handle(msg, devlink))
4666                 goto genlmsg_cancel;
4667
4668         if (cmd == DEVLINK_CMD_PORT_PARAM_GET ||
4669             cmd == DEVLINK_CMD_PORT_PARAM_NEW ||
4670             cmd == DEVLINK_CMD_PORT_PARAM_DEL)
4671                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, port_index))
4672                         goto genlmsg_cancel;
4673
4674         param_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_PARAM);
4675         if (!param_attr)
4676                 goto genlmsg_cancel;
4677         if (nla_put_string(msg, DEVLINK_ATTR_PARAM_NAME, param->name))
4678                 goto param_nest_cancel;
4679         if (param->generic && nla_put_flag(msg, DEVLINK_ATTR_PARAM_GENERIC))
4680                 goto param_nest_cancel;
4681
4682         nla_type = devlink_param_type_to_nla_type(param->type);
4683         if (nla_type < 0)
4684                 goto param_nest_cancel;
4685         if (nla_put_u8(msg, DEVLINK_ATTR_PARAM_TYPE, nla_type))
4686                 goto param_nest_cancel;
4687
4688         param_values_list = nla_nest_start_noflag(msg,
4689                                                   DEVLINK_ATTR_PARAM_VALUES_LIST);
4690         if (!param_values_list)
4691                 goto param_nest_cancel;
4692
4693         for (i = 0; i <= DEVLINK_PARAM_CMODE_MAX; i++) {
4694                 if (!param_value_set[i])
4695                         continue;
4696                 err = devlink_nl_param_value_fill_one(msg, param->type,
4697                                                       i, param_value[i]);
4698                 if (err)
4699                         goto values_list_nest_cancel;
4700         }
4701
4702         nla_nest_end(msg, param_values_list);
4703         nla_nest_end(msg, param_attr);
4704         genlmsg_end(msg, hdr);
4705         return 0;
4706
4707 values_list_nest_cancel:
4708         nla_nest_end(msg, param_values_list);
4709 param_nest_cancel:
4710         nla_nest_cancel(msg, param_attr);
4711 genlmsg_cancel:
4712         genlmsg_cancel(msg, hdr);
4713         return -EMSGSIZE;
4714 }
4715
4716 static void devlink_param_notify(struct devlink *devlink,
4717                                  unsigned int port_index,
4718                                  struct devlink_param_item *param_item,
4719                                  enum devlink_command cmd)
4720 {
4721         struct sk_buff *msg;
4722         int err;
4723
4724         WARN_ON(cmd != DEVLINK_CMD_PARAM_NEW && cmd != DEVLINK_CMD_PARAM_DEL &&
4725                 cmd != DEVLINK_CMD_PORT_PARAM_NEW &&
4726                 cmd != DEVLINK_CMD_PORT_PARAM_DEL);
4727         ASSERT_DEVLINK_REGISTERED(devlink);
4728
4729         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4730         if (!msg)
4731                 return;
4732         err = devlink_nl_param_fill(msg, devlink, port_index, param_item, cmd,
4733                                     0, 0, 0);
4734         if (err) {
4735                 nlmsg_free(msg);
4736                 return;
4737         }
4738
4739         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
4740                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
4741 }
4742
4743 static int devlink_nl_cmd_param_get_dumpit(struct sk_buff *msg,
4744                                            struct netlink_callback *cb)
4745 {
4746         struct devlink_param_item *param_item;
4747         struct devlink *devlink;
4748         int start = cb->args[0];
4749         unsigned long index;
4750         int idx = 0;
4751         int err = 0;
4752
4753         mutex_lock(&devlink_mutex);
4754         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
4755                 if (!devlink_try_get(devlink))
4756                         continue;
4757
4758                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4759                         goto retry;
4760
4761                 mutex_lock(&devlink->lock);
4762                 list_for_each_entry(param_item, &devlink->param_list, list) {
4763                         if (idx < start) {
4764                                 idx++;
4765                                 continue;
4766                         }
4767                         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
4768                                                     DEVLINK_CMD_PARAM_GET,
4769                                                     NETLINK_CB(cb->skb).portid,
4770                                                     cb->nlh->nlmsg_seq,
4771                                                     NLM_F_MULTI);
4772                         if (err == -EOPNOTSUPP) {
4773                                 err = 0;
4774                         } else if (err) {
4775                                 mutex_unlock(&devlink->lock);
4776                                 devlink_put(devlink);
4777                                 goto out;
4778                         }
4779                         idx++;
4780                 }
4781                 mutex_unlock(&devlink->lock);
4782 retry:
4783                 devlink_put(devlink);
4784         }
4785 out:
4786         mutex_unlock(&devlink_mutex);
4787
4788         if (err != -EMSGSIZE)
4789                 return err;
4790
4791         cb->args[0] = idx;
4792         return msg->len;
4793 }
4794
4795 static int
4796 devlink_param_type_get_from_info(struct genl_info *info,
4797                                  enum devlink_param_type *param_type)
4798 {
4799         if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
4800                 return -EINVAL;
4801
4802         switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
4803         case NLA_U8:
4804                 *param_type = DEVLINK_PARAM_TYPE_U8;
4805                 break;
4806         case NLA_U16:
4807                 *param_type = DEVLINK_PARAM_TYPE_U16;
4808                 break;
4809         case NLA_U32:
4810                 *param_type = DEVLINK_PARAM_TYPE_U32;
4811                 break;
4812         case NLA_STRING:
4813                 *param_type = DEVLINK_PARAM_TYPE_STRING;
4814                 break;
4815         case NLA_FLAG:
4816                 *param_type = DEVLINK_PARAM_TYPE_BOOL;
4817                 break;
4818         default:
4819                 return -EINVAL;
4820         }
4821
4822         return 0;
4823 }
4824
4825 static int
4826 devlink_param_value_get_from_info(const struct devlink_param *param,
4827                                   struct genl_info *info,
4828                                   union devlink_param_value *value)
4829 {
4830         struct nlattr *param_data;
4831         int len;
4832
4833         param_data = info->attrs[DEVLINK_ATTR_PARAM_VALUE_DATA];
4834
4835         if (param->type != DEVLINK_PARAM_TYPE_BOOL && !param_data)
4836                 return -EINVAL;
4837
4838         switch (param->type) {
4839         case DEVLINK_PARAM_TYPE_U8:
4840                 if (nla_len(param_data) != sizeof(u8))
4841                         return -EINVAL;
4842                 value->vu8 = nla_get_u8(param_data);
4843                 break;
4844         case DEVLINK_PARAM_TYPE_U16:
4845                 if (nla_len(param_data) != sizeof(u16))
4846                         return -EINVAL;
4847                 value->vu16 = nla_get_u16(param_data);
4848                 break;
4849         case DEVLINK_PARAM_TYPE_U32:
4850                 if (nla_len(param_data) != sizeof(u32))
4851                         return -EINVAL;
4852                 value->vu32 = nla_get_u32(param_data);
4853                 break;
4854         case DEVLINK_PARAM_TYPE_STRING:
4855                 len = strnlen(nla_data(param_data), nla_len(param_data));
4856                 if (len == nla_len(param_data) ||
4857                     len >= __DEVLINK_PARAM_MAX_STRING_VALUE)
4858                         return -EINVAL;
4859                 strcpy(value->vstr, nla_data(param_data));
4860                 break;
4861         case DEVLINK_PARAM_TYPE_BOOL:
4862                 if (param_data && nla_len(param_data))
4863                         return -EINVAL;
4864                 value->vbool = nla_get_flag(param_data);
4865                 break;
4866         }
4867         return 0;
4868 }
4869
4870 static struct devlink_param_item *
4871 devlink_param_get_from_info(struct list_head *param_list,
4872                             struct genl_info *info)
4873 {
4874         char *param_name;
4875
4876         if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
4877                 return NULL;
4878
4879         param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
4880         return devlink_param_find_by_name(param_list, param_name);
4881 }
4882
4883 static int devlink_nl_cmd_param_get_doit(struct sk_buff *skb,
4884                                          struct genl_info *info)
4885 {
4886         struct devlink *devlink = info->user_ptr[0];
4887         struct devlink_param_item *param_item;
4888         struct sk_buff *msg;
4889         int err;
4890
4891         param_item = devlink_param_get_from_info(&devlink->param_list, info);
4892         if (!param_item)
4893                 return -EINVAL;
4894
4895         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
4896         if (!msg)
4897                 return -ENOMEM;
4898
4899         err = devlink_nl_param_fill(msg, devlink, 0, param_item,
4900                                     DEVLINK_CMD_PARAM_GET,
4901                                     info->snd_portid, info->snd_seq, 0);
4902         if (err) {
4903                 nlmsg_free(msg);
4904                 return err;
4905         }
4906
4907         return genlmsg_reply(msg, info);
4908 }
4909
4910 static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
4911                                            unsigned int port_index,
4912                                            struct list_head *param_list,
4913                                            struct genl_info *info,
4914                                            enum devlink_command cmd)
4915 {
4916         enum devlink_param_type param_type;
4917         struct devlink_param_gset_ctx ctx;
4918         enum devlink_param_cmode cmode;
4919         struct devlink_param_item *param_item;
4920         const struct devlink_param *param;
4921         union devlink_param_value value;
4922         int err = 0;
4923
4924         param_item = devlink_param_get_from_info(param_list, info);
4925         if (!param_item)
4926                 return -EINVAL;
4927         param = param_item->param;
4928         err = devlink_param_type_get_from_info(info, &param_type);
4929         if (err)
4930                 return err;
4931         if (param_type != param->type)
4932                 return -EINVAL;
4933         err = devlink_param_value_get_from_info(param, info, &value);
4934         if (err)
4935                 return err;
4936         if (param->validate) {
4937                 err = param->validate(devlink, param->id, value, info->extack);
4938                 if (err)
4939                         return err;
4940         }
4941
4942         if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
4943                 return -EINVAL;
4944         cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
4945         if (!devlink_param_cmode_is_supported(param, cmode))
4946                 return -EOPNOTSUPP;
4947
4948         if (cmode == DEVLINK_PARAM_CMODE_DRIVERINIT) {
4949                 if (param->type == DEVLINK_PARAM_TYPE_STRING)
4950                         strcpy(param_item->driverinit_value.vstr, value.vstr);
4951                 else
4952                         param_item->driverinit_value = value;
4953                 param_item->driverinit_value_valid = true;
4954         } else {
4955                 if (!param->set)
4956                         return -EOPNOTSUPP;
4957                 ctx.val = value;
4958                 ctx.cmode = cmode;
4959                 err = devlink_param_set(devlink, param, &ctx);
4960                 if (err)
4961                         return err;
4962         }
4963
4964         devlink_param_notify(devlink, port_index, param_item, cmd);
4965         return 0;
4966 }
4967
4968 static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
4969                                          struct genl_info *info)
4970 {
4971         struct devlink *devlink = info->user_ptr[0];
4972
4973         return __devlink_nl_cmd_param_set_doit(devlink, 0, &devlink->param_list,
4974                                                info, DEVLINK_CMD_PARAM_NEW);
4975 }
4976
4977 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
4978                                                 struct netlink_callback *cb)
4979 {
4980         struct devlink_param_item *param_item;
4981         struct devlink_port *devlink_port;
4982         struct devlink *devlink;
4983         int start = cb->args[0];
4984         unsigned long index;
4985         int idx = 0;
4986         int err = 0;
4987
4988         mutex_lock(&devlink_mutex);
4989         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
4990                 if (!devlink_try_get(devlink))
4991                         continue;
4992
4993                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
4994                         goto retry;
4995
4996                 mutex_lock(&devlink->lock);
4997                 list_for_each_entry(devlink_port, &devlink->port_list, list) {
4998                         list_for_each_entry(param_item,
4999                                             &devlink_port->param_list, list) {
5000                                 if (idx < start) {
5001                                         idx++;
5002                                         continue;
5003                                 }
5004                                 err = devlink_nl_param_fill(msg,
5005                                                 devlink_port->devlink,
5006                                                 devlink_port->index, param_item,
5007                                                 DEVLINK_CMD_PORT_PARAM_GET,
5008                                                 NETLINK_CB(cb->skb).portid,
5009                                                 cb->nlh->nlmsg_seq,
5010                                                 NLM_F_MULTI);
5011                                 if (err == -EOPNOTSUPP) {
5012                                         err = 0;
5013                                 } else if (err) {
5014                                         mutex_unlock(&devlink->lock);
5015                                         devlink_put(devlink);
5016                                         goto out;
5017                                 }
5018                                 idx++;
5019                         }
5020                 }
5021                 mutex_unlock(&devlink->lock);
5022 retry:
5023                 devlink_put(devlink);
5024         }
5025 out:
5026         mutex_unlock(&devlink_mutex);
5027
5028         if (err != -EMSGSIZE)
5029                 return err;
5030
5031         cb->args[0] = idx;
5032         return msg->len;
5033 }
5034
5035 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
5036                                               struct genl_info *info)
5037 {
5038         struct devlink_port *devlink_port = info->user_ptr[1];
5039         struct devlink_param_item *param_item;
5040         struct sk_buff *msg;
5041         int err;
5042
5043         param_item = devlink_param_get_from_info(&devlink_port->param_list,
5044                                                  info);
5045         if (!param_item)
5046                 return -EINVAL;
5047
5048         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5049         if (!msg)
5050                 return -ENOMEM;
5051
5052         err = devlink_nl_param_fill(msg, devlink_port->devlink,
5053                                     devlink_port->index, param_item,
5054                                     DEVLINK_CMD_PORT_PARAM_GET,
5055                                     info->snd_portid, info->snd_seq, 0);
5056         if (err) {
5057                 nlmsg_free(msg);
5058                 return err;
5059         }
5060
5061         return genlmsg_reply(msg, info);
5062 }
5063
5064 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
5065                                               struct genl_info *info)
5066 {
5067         struct devlink_port *devlink_port = info->user_ptr[1];
5068
5069         return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
5070                                                devlink_port->index,
5071                                                &devlink_port->param_list, info,
5072                                                DEVLINK_CMD_PORT_PARAM_NEW);
5073 }
5074
5075 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
5076                                              struct devlink *devlink,
5077                                              struct devlink_snapshot *snapshot)
5078 {
5079         struct nlattr *snap_attr;
5080         int err;
5081
5082         snap_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_SNAPSHOT);
5083         if (!snap_attr)
5084                 return -EINVAL;
5085
5086         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID, snapshot->id);
5087         if (err)
5088                 goto nla_put_failure;
5089
5090         nla_nest_end(msg, snap_attr);
5091         return 0;
5092
5093 nla_put_failure:
5094         nla_nest_cancel(msg, snap_attr);
5095         return err;
5096 }
5097
5098 static int devlink_nl_region_snapshots_id_put(struct sk_buff *msg,
5099                                               struct devlink *devlink,
5100                                               struct devlink_region *region)
5101 {
5102         struct devlink_snapshot *snapshot;
5103         struct nlattr *snapshots_attr;
5104         int err;
5105
5106         snapshots_attr = nla_nest_start_noflag(msg,
5107                                                DEVLINK_ATTR_REGION_SNAPSHOTS);
5108         if (!snapshots_attr)
5109                 return -EINVAL;
5110
5111         list_for_each_entry(snapshot, &region->snapshot_list, list) {
5112                 err = devlink_nl_region_snapshot_id_put(msg, devlink, snapshot);
5113                 if (err)
5114                         goto nla_put_failure;
5115         }
5116
5117         nla_nest_end(msg, snapshots_attr);
5118         return 0;
5119
5120 nla_put_failure:
5121         nla_nest_cancel(msg, snapshots_attr);
5122         return err;
5123 }
5124
5125 static int devlink_nl_region_fill(struct sk_buff *msg, struct devlink *devlink,
5126                                   enum devlink_command cmd, u32 portid,
5127                                   u32 seq, int flags,
5128                                   struct devlink_region *region)
5129 {
5130         void *hdr;
5131         int err;
5132
5133         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
5134         if (!hdr)
5135                 return -EMSGSIZE;
5136
5137         err = devlink_nl_put_handle(msg, devlink);
5138         if (err)
5139                 goto nla_put_failure;
5140
5141         if (region->port) {
5142                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5143                                   region->port->index);
5144                 if (err)
5145                         goto nla_put_failure;
5146         }
5147
5148         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME, region->ops->name);
5149         if (err)
5150                 goto nla_put_failure;
5151
5152         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5153                                 region->size,
5154                                 DEVLINK_ATTR_PAD);
5155         if (err)
5156                 goto nla_put_failure;
5157
5158         err = nla_put_u32(msg, DEVLINK_ATTR_REGION_MAX_SNAPSHOTS,
5159                           region->max_snapshots);
5160         if (err)
5161                 goto nla_put_failure;
5162
5163         err = devlink_nl_region_snapshots_id_put(msg, devlink, region);
5164         if (err)
5165                 goto nla_put_failure;
5166
5167         genlmsg_end(msg, hdr);
5168         return 0;
5169
5170 nla_put_failure:
5171         genlmsg_cancel(msg, hdr);
5172         return err;
5173 }
5174
5175 static struct sk_buff *
5176 devlink_nl_region_notify_build(struct devlink_region *region,
5177                                struct devlink_snapshot *snapshot,
5178                                enum devlink_command cmd, u32 portid, u32 seq)
5179 {
5180         struct devlink *devlink = region->devlink;
5181         struct sk_buff *msg;
5182         void *hdr;
5183         int err;
5184
5185
5186         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5187         if (!msg)
5188                 return ERR_PTR(-ENOMEM);
5189
5190         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, 0, cmd);
5191         if (!hdr) {
5192                 err = -EMSGSIZE;
5193                 goto out_free_msg;
5194         }
5195
5196         err = devlink_nl_put_handle(msg, devlink);
5197         if (err)
5198                 goto out_cancel_msg;
5199
5200         if (region->port) {
5201                 err = nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX,
5202                                   region->port->index);
5203                 if (err)
5204                         goto out_cancel_msg;
5205         }
5206
5207         err = nla_put_string(msg, DEVLINK_ATTR_REGION_NAME,
5208                              region->ops->name);
5209         if (err)
5210                 goto out_cancel_msg;
5211
5212         if (snapshot) {
5213                 err = nla_put_u32(msg, DEVLINK_ATTR_REGION_SNAPSHOT_ID,
5214                                   snapshot->id);
5215                 if (err)
5216                         goto out_cancel_msg;
5217         } else {
5218                 err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_SIZE,
5219                                         region->size, DEVLINK_ATTR_PAD);
5220                 if (err)
5221                         goto out_cancel_msg;
5222         }
5223         genlmsg_end(msg, hdr);
5224
5225         return msg;
5226
5227 out_cancel_msg:
5228         genlmsg_cancel(msg, hdr);
5229 out_free_msg:
5230         nlmsg_free(msg);
5231         return ERR_PTR(err);
5232 }
5233
5234 static void devlink_nl_region_notify(struct devlink_region *region,
5235                                      struct devlink_snapshot *snapshot,
5236                                      enum devlink_command cmd)
5237 {
5238         struct devlink *devlink = region->devlink;
5239         struct sk_buff *msg;
5240
5241         WARN_ON(cmd != DEVLINK_CMD_REGION_NEW && cmd != DEVLINK_CMD_REGION_DEL);
5242         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
5243                 return;
5244
5245         msg = devlink_nl_region_notify_build(region, snapshot, cmd, 0, 0);
5246         if (IS_ERR(msg))
5247                 return;
5248
5249         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
5250                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
5251 }
5252
5253 /**
5254  * __devlink_snapshot_id_increment - Increment number of snapshots using an id
5255  *      @devlink: devlink instance
5256  *      @id: the snapshot id
5257  *
5258  *      Track when a new snapshot begins using an id. Load the count for the
5259  *      given id from the snapshot xarray, increment it, and store it back.
5260  *
5261  *      Called when a new snapshot is created with the given id.
5262  *
5263  *      The id *must* have been previously allocated by
5264  *      devlink_region_snapshot_id_get().
5265  *
5266  *      Returns 0 on success, or an error on failure.
5267  */
5268 static int __devlink_snapshot_id_increment(struct devlink *devlink, u32 id)
5269 {
5270         unsigned long count;
5271         void *p;
5272
5273         lockdep_assert_held(&devlink->lock);
5274
5275         p = xa_load(&devlink->snapshot_ids, id);
5276         if (WARN_ON(!p))
5277                 return -EINVAL;
5278
5279         if (WARN_ON(!xa_is_value(p)))
5280                 return -EINVAL;
5281
5282         count = xa_to_value(p);
5283         count++;
5284
5285         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5286                                GFP_KERNEL));
5287 }
5288
5289 /**
5290  * __devlink_snapshot_id_decrement - Decrease number of snapshots using an id
5291  *      @devlink: devlink instance
5292  *      @id: the snapshot id
5293  *
5294  *      Track when a snapshot is deleted and stops using an id. Load the count
5295  *      for the given id from the snapshot xarray, decrement it, and store it
5296  *      back.
5297  *
5298  *      If the count reaches zero, erase this id from the xarray, freeing it
5299  *      up for future re-use by devlink_region_snapshot_id_get().
5300  *
5301  *      Called when a snapshot using the given id is deleted, and when the
5302  *      initial allocator of the id is finished using it.
5303  */
5304 static void __devlink_snapshot_id_decrement(struct devlink *devlink, u32 id)
5305 {
5306         unsigned long count;
5307         void *p;
5308
5309         lockdep_assert_held(&devlink->lock);
5310
5311         p = xa_load(&devlink->snapshot_ids, id);
5312         if (WARN_ON(!p))
5313                 return;
5314
5315         if (WARN_ON(!xa_is_value(p)))
5316                 return;
5317
5318         count = xa_to_value(p);
5319
5320         if (count > 1) {
5321                 count--;
5322                 xa_store(&devlink->snapshot_ids, id, xa_mk_value(count),
5323                          GFP_KERNEL);
5324         } else {
5325                 /* If this was the last user, we can erase this id */
5326                 xa_erase(&devlink->snapshot_ids, id);
5327         }
5328 }
5329
5330 /**
5331  *      __devlink_snapshot_id_insert - Insert a specific snapshot ID
5332  *      @devlink: devlink instance
5333  *      @id: the snapshot id
5334  *
5335  *      Mark the given snapshot id as used by inserting a zero value into the
5336  *      snapshot xarray.
5337  *
5338  *      This must be called while holding the devlink instance lock. Unlike
5339  *      devlink_snapshot_id_get, the initial reference count is zero, not one.
5340  *      It is expected that the id will immediately be used before
5341  *      releasing the devlink instance lock.
5342  *
5343  *      Returns zero on success, or an error code if the snapshot id could not
5344  *      be inserted.
5345  */
5346 static int __devlink_snapshot_id_insert(struct devlink *devlink, u32 id)
5347 {
5348         lockdep_assert_held(&devlink->lock);
5349
5350         if (xa_load(&devlink->snapshot_ids, id))
5351                 return -EEXIST;
5352
5353         return xa_err(xa_store(&devlink->snapshot_ids, id, xa_mk_value(0),
5354                                GFP_KERNEL));
5355 }
5356
5357 /**
5358  *      __devlink_region_snapshot_id_get - get snapshot ID
5359  *      @devlink: devlink instance
5360  *      @id: storage to return snapshot id
5361  *
5362  *      Allocates a new snapshot id. Returns zero on success, or a negative
5363  *      error on failure. Must be called while holding the devlink instance
5364  *      lock.
5365  *
5366  *      Snapshot IDs are tracked using an xarray which stores the number of
5367  *      users of the snapshot id.
5368  *
5369  *      Note that the caller of this function counts as a 'user', in order to
5370  *      avoid race conditions. The caller must release its hold on the
5371  *      snapshot by using devlink_region_snapshot_id_put.
5372  */
5373 static int __devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
5374 {
5375         lockdep_assert_held(&devlink->lock);
5376
5377         return xa_alloc(&devlink->snapshot_ids, id, xa_mk_value(1),
5378                         xa_limit_32b, GFP_KERNEL);
5379 }
5380
5381 /**
5382  *      __devlink_region_snapshot_create - create a new snapshot
5383  *      This will add a new snapshot of a region. The snapshot
5384  *      will be stored on the region struct and can be accessed
5385  *      from devlink. This is useful for future analyses of snapshots.
5386  *      Multiple snapshots can be created on a region.
5387  *      The @snapshot_id should be obtained using the getter function.
5388  *
5389  *      Must be called only while holding the devlink instance lock.
5390  *
5391  *      @region: devlink region of the snapshot
5392  *      @data: snapshot data
5393  *      @snapshot_id: snapshot id to be created
5394  */
5395 static int
5396 __devlink_region_snapshot_create(struct devlink_region *region,
5397                                  u8 *data, u32 snapshot_id)
5398 {
5399         struct devlink *devlink = region->devlink;
5400         struct devlink_snapshot *snapshot;
5401         int err;
5402
5403         lockdep_assert_held(&devlink->lock);
5404
5405         /* check if region can hold one more snapshot */
5406         if (region->cur_snapshots == region->max_snapshots)
5407                 return -ENOSPC;
5408
5409         if (devlink_region_snapshot_get_by_id(region, snapshot_id))
5410                 return -EEXIST;
5411
5412         snapshot = kzalloc(sizeof(*snapshot), GFP_KERNEL);
5413         if (!snapshot)
5414                 return -ENOMEM;
5415
5416         err = __devlink_snapshot_id_increment(devlink, snapshot_id);
5417         if (err)
5418                 goto err_snapshot_id_increment;
5419
5420         snapshot->id = snapshot_id;
5421         snapshot->region = region;
5422         snapshot->data = data;
5423
5424         list_add_tail(&snapshot->list, &region->snapshot_list);
5425
5426         region->cur_snapshots++;
5427
5428         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_NEW);
5429         return 0;
5430
5431 err_snapshot_id_increment:
5432         kfree(snapshot);
5433         return err;
5434 }
5435
5436 static void devlink_region_snapshot_del(struct devlink_region *region,
5437                                         struct devlink_snapshot *snapshot)
5438 {
5439         struct devlink *devlink = region->devlink;
5440
5441         lockdep_assert_held(&devlink->lock);
5442
5443         devlink_nl_region_notify(region, snapshot, DEVLINK_CMD_REGION_DEL);
5444         region->cur_snapshots--;
5445         list_del(&snapshot->list);
5446         region->ops->destructor(snapshot->data);
5447         __devlink_snapshot_id_decrement(devlink, snapshot->id);
5448         kfree(snapshot);
5449 }
5450
5451 static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
5452                                           struct genl_info *info)
5453 {
5454         struct devlink *devlink = info->user_ptr[0];
5455         struct devlink_port *port = NULL;
5456         struct devlink_region *region;
5457         const char *region_name;
5458         struct sk_buff *msg;
5459         unsigned int index;
5460         int err;
5461
5462         if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
5463                 return -EINVAL;
5464
5465         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5466                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5467
5468                 port = devlink_port_get_by_index(devlink, index);
5469                 if (!port)
5470                         return -ENODEV;
5471         }
5472
5473         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5474         if (port)
5475                 region = devlink_port_region_get_by_name(port, region_name);
5476         else
5477                 region = devlink_region_get_by_name(devlink, region_name);
5478
5479         if (!region)
5480                 return -EINVAL;
5481
5482         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
5483         if (!msg)
5484                 return -ENOMEM;
5485
5486         err = devlink_nl_region_fill(msg, devlink, DEVLINK_CMD_REGION_GET,
5487                                      info->snd_portid, info->snd_seq, 0,
5488                                      region);
5489         if (err) {
5490                 nlmsg_free(msg);
5491                 return err;
5492         }
5493
5494         return genlmsg_reply(msg, info);
5495 }
5496
5497 static int devlink_nl_cmd_region_get_port_dumpit(struct sk_buff *msg,
5498                                                  struct netlink_callback *cb,
5499                                                  struct devlink_port *port,
5500                                                  int *idx,
5501                                                  int start)
5502 {
5503         struct devlink_region *region;
5504         int err = 0;
5505
5506         list_for_each_entry(region, &port->region_list, list) {
5507                 if (*idx < start) {
5508                         (*idx)++;
5509                         continue;
5510                 }
5511                 err = devlink_nl_region_fill(msg, port->devlink,
5512                                              DEVLINK_CMD_REGION_GET,
5513                                              NETLINK_CB(cb->skb).portid,
5514                                              cb->nlh->nlmsg_seq,
5515                                              NLM_F_MULTI, region);
5516                 if (err)
5517                         goto out;
5518                 (*idx)++;
5519         }
5520
5521 out:
5522         return err;
5523 }
5524
5525 static int devlink_nl_cmd_region_get_devlink_dumpit(struct sk_buff *msg,
5526                                                     struct netlink_callback *cb,
5527                                                     struct devlink *devlink,
5528                                                     int *idx,
5529                                                     int start)
5530 {
5531         struct devlink_region *region;
5532         struct devlink_port *port;
5533         int err = 0;
5534
5535         mutex_lock(&devlink->lock);
5536         list_for_each_entry(region, &devlink->region_list, list) {
5537                 if (*idx < start) {
5538                         (*idx)++;
5539                         continue;
5540                 }
5541                 err = devlink_nl_region_fill(msg, devlink,
5542                                              DEVLINK_CMD_REGION_GET,
5543                                              NETLINK_CB(cb->skb).portid,
5544                                              cb->nlh->nlmsg_seq,
5545                                              NLM_F_MULTI, region);
5546                 if (err)
5547                         goto out;
5548                 (*idx)++;
5549         }
5550
5551         list_for_each_entry(port, &devlink->port_list, list) {
5552                 err = devlink_nl_cmd_region_get_port_dumpit(msg, cb, port, idx,
5553                                                             start);
5554                 if (err)
5555                         goto out;
5556         }
5557
5558 out:
5559         mutex_unlock(&devlink->lock);
5560         return err;
5561 }
5562
5563 static int devlink_nl_cmd_region_get_dumpit(struct sk_buff *msg,
5564                                             struct netlink_callback *cb)
5565 {
5566         struct devlink *devlink;
5567         int start = cb->args[0];
5568         unsigned long index;
5569         int idx = 0;
5570         int err = 0;
5571
5572         mutex_lock(&devlink_mutex);
5573         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
5574                 if (!devlink_try_get(devlink))
5575                         continue;
5576
5577                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
5578                         goto retry;
5579
5580                 err = devlink_nl_cmd_region_get_devlink_dumpit(msg, cb, devlink,
5581                                                                &idx, start);
5582 retry:
5583                 devlink_put(devlink);
5584                 if (err)
5585                         goto out;
5586         }
5587 out:
5588         mutex_unlock(&devlink_mutex);
5589         cb->args[0] = idx;
5590         return msg->len;
5591 }
5592
5593 static int devlink_nl_cmd_region_del(struct sk_buff *skb,
5594                                      struct genl_info *info)
5595 {
5596         struct devlink *devlink = info->user_ptr[0];
5597         struct devlink_snapshot *snapshot;
5598         struct devlink_port *port = NULL;
5599         struct devlink_region *region;
5600         const char *region_name;
5601         unsigned int index;
5602         u32 snapshot_id;
5603
5604         if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
5605             !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
5606                 return -EINVAL;
5607
5608         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5609         snapshot_id = nla_get_u32(info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
5610
5611         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5612                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5613
5614                 port = devlink_port_get_by_index(devlink, index);
5615                 if (!port)
5616                         return -ENODEV;
5617         }
5618
5619         if (port)
5620                 region = devlink_port_region_get_by_name(port, region_name);
5621         else
5622                 region = devlink_region_get_by_name(devlink, region_name);
5623
5624         if (!region)
5625                 return -EINVAL;
5626
5627         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
5628         if (!snapshot)
5629                 return -EINVAL;
5630
5631         devlink_region_snapshot_del(region, snapshot);
5632         return 0;
5633 }
5634
5635 static int
5636 devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
5637 {
5638         struct devlink *devlink = info->user_ptr[0];
5639         struct devlink_snapshot *snapshot;
5640         struct devlink_port *port = NULL;
5641         struct nlattr *snapshot_id_attr;
5642         struct devlink_region *region;
5643         const char *region_name;
5644         unsigned int index;
5645         u32 snapshot_id;
5646         u8 *data;
5647         int err;
5648
5649         if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
5650                 NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
5651                 return -EINVAL;
5652         }
5653
5654         region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
5655
5656         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5657                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5658
5659                 port = devlink_port_get_by_index(devlink, index);
5660                 if (!port)
5661                         return -ENODEV;
5662         }
5663
5664         if (port)
5665                 region = devlink_port_region_get_by_name(port, region_name);
5666         else
5667                 region = devlink_region_get_by_name(devlink, region_name);
5668
5669         if (!region) {
5670                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not exist");
5671                 return -EINVAL;
5672         }
5673
5674         if (!region->ops->snapshot) {
5675                 NL_SET_ERR_MSG_MOD(info->extack, "The requested region does not support taking an immediate snapshot");
5676                 return -EOPNOTSUPP;
5677         }
5678
5679         if (region->cur_snapshots == region->max_snapshots) {
5680                 NL_SET_ERR_MSG_MOD(info->extack, "The region has reached the maximum number of stored snapshots");
5681                 return -ENOSPC;
5682         }
5683
5684         snapshot_id_attr = info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID];
5685         if (snapshot_id_attr) {
5686                 snapshot_id = nla_get_u32(snapshot_id_attr);
5687
5688                 if (devlink_region_snapshot_get_by_id(region, snapshot_id)) {
5689                         NL_SET_ERR_MSG_MOD(info->extack, "The requested snapshot id is already in use");
5690                         return -EEXIST;
5691                 }
5692
5693                 err = __devlink_snapshot_id_insert(devlink, snapshot_id);
5694                 if (err)
5695                         return err;
5696         } else {
5697                 err = __devlink_region_snapshot_id_get(devlink, &snapshot_id);
5698                 if (err) {
5699                         NL_SET_ERR_MSG_MOD(info->extack, "Failed to allocate a new snapshot id");
5700                         return err;
5701                 }
5702         }
5703
5704         if (port)
5705                 err = region->port_ops->snapshot(port, region->port_ops,
5706                                                  info->extack, &data);
5707         else
5708                 err = region->ops->snapshot(devlink, region->ops,
5709                                             info->extack, &data);
5710         if (err)
5711                 goto err_snapshot_capture;
5712
5713         err = __devlink_region_snapshot_create(region, data, snapshot_id);
5714         if (err)
5715                 goto err_snapshot_create;
5716
5717         if (!snapshot_id_attr) {
5718                 struct sk_buff *msg;
5719
5720                 snapshot = devlink_region_snapshot_get_by_id(region,
5721                                                              snapshot_id);
5722                 if (WARN_ON(!snapshot))
5723                         return -EINVAL;
5724
5725                 msg = devlink_nl_region_notify_build(region, snapshot,
5726                                                      DEVLINK_CMD_REGION_NEW,
5727                                                      info->snd_portid,
5728                                                      info->snd_seq);
5729                 err = PTR_ERR_OR_ZERO(msg);
5730                 if (err)
5731                         goto err_notify;
5732
5733                 err = genlmsg_reply(msg, info);
5734                 if (err)
5735                         goto err_notify;
5736         }
5737
5738         return 0;
5739
5740 err_snapshot_create:
5741         region->ops->destructor(data);
5742 err_snapshot_capture:
5743         __devlink_snapshot_id_decrement(devlink, snapshot_id);
5744         return err;
5745
5746 err_notify:
5747         devlink_region_snapshot_del(region, snapshot);
5748         return err;
5749 }
5750
5751 static int devlink_nl_cmd_region_read_chunk_fill(struct sk_buff *msg,
5752                                                  struct devlink *devlink,
5753                                                  u8 *chunk, u32 chunk_size,
5754                                                  u64 addr)
5755 {
5756         struct nlattr *chunk_attr;
5757         int err;
5758
5759         chunk_attr = nla_nest_start_noflag(msg, DEVLINK_ATTR_REGION_CHUNK);
5760         if (!chunk_attr)
5761                 return -EINVAL;
5762
5763         err = nla_put(msg, DEVLINK_ATTR_REGION_CHUNK_DATA, chunk_size, chunk);
5764         if (err)
5765                 goto nla_put_failure;
5766
5767         err = nla_put_u64_64bit(msg, DEVLINK_ATTR_REGION_CHUNK_ADDR, addr,
5768                                 DEVLINK_ATTR_PAD);
5769         if (err)
5770                 goto nla_put_failure;
5771
5772         nla_nest_end(msg, chunk_attr);
5773         return 0;
5774
5775 nla_put_failure:
5776         nla_nest_cancel(msg, chunk_attr);
5777         return err;
5778 }
5779
5780 #define DEVLINK_REGION_READ_CHUNK_SIZE 256
5781
5782 static int devlink_nl_region_read_snapshot_fill(struct sk_buff *skb,
5783                                                 struct devlink *devlink,
5784                                                 struct devlink_region *region,
5785                                                 struct nlattr **attrs,
5786                                                 u64 start_offset,
5787                                                 u64 end_offset,
5788                                                 u64 *new_offset)
5789 {
5790         struct devlink_snapshot *snapshot;
5791         u64 curr_offset = start_offset;
5792         u32 snapshot_id;
5793         int err = 0;
5794
5795         *new_offset = start_offset;
5796
5797         snapshot_id = nla_get_u32(attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]);
5798         snapshot = devlink_region_snapshot_get_by_id(region, snapshot_id);
5799         if (!snapshot)
5800                 return -EINVAL;
5801
5802         while (curr_offset < end_offset) {
5803                 u32 data_size;
5804                 u8 *data;
5805
5806                 if (end_offset - curr_offset < DEVLINK_REGION_READ_CHUNK_SIZE)
5807                         data_size = end_offset - curr_offset;
5808                 else
5809                         data_size = DEVLINK_REGION_READ_CHUNK_SIZE;
5810
5811                 data = &snapshot->data[curr_offset];
5812                 err = devlink_nl_cmd_region_read_chunk_fill(skb, devlink,
5813                                                             data, data_size,
5814                                                             curr_offset);
5815                 if (err)
5816                         break;
5817
5818                 curr_offset += data_size;
5819         }
5820         *new_offset = curr_offset;
5821
5822         return err;
5823 }
5824
5825 static int devlink_nl_cmd_region_read_dumpit(struct sk_buff *skb,
5826                                              struct netlink_callback *cb)
5827 {
5828         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
5829         u64 ret_offset, start_offset, end_offset = U64_MAX;
5830         struct nlattr **attrs = info->attrs;
5831         struct devlink_port *port = NULL;
5832         struct devlink_region *region;
5833         struct nlattr *chunks_attr;
5834         const char *region_name;
5835         struct devlink *devlink;
5836         unsigned int index;
5837         void *hdr;
5838         int err;
5839
5840         start_offset = *((u64 *)&cb->args[0]);
5841
5842         mutex_lock(&devlink_mutex);
5843         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
5844         if (IS_ERR(devlink)) {
5845                 err = PTR_ERR(devlink);
5846                 goto out_dev;
5847         }
5848
5849         mutex_lock(&devlink->lock);
5850
5851         if (!attrs[DEVLINK_ATTR_REGION_NAME] ||
5852             !attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) {
5853                 err = -EINVAL;
5854                 goto out_unlock;
5855         }
5856
5857         if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
5858                 index = nla_get_u32(info->attrs[DEVLINK_ATTR_PORT_INDEX]);
5859
5860                 port = devlink_port_get_by_index(devlink, index);
5861                 if (!port) {
5862                         err = -ENODEV;
5863                         goto out_unlock;
5864                 }
5865         }
5866
5867         region_name = nla_data(attrs[DEVLINK_ATTR_REGION_NAME]);
5868
5869         if (port)
5870                 region = devlink_port_region_get_by_name(port, region_name);
5871         else
5872                 region = devlink_region_get_by_name(devlink, region_name);
5873
5874         if (!region) {
5875                 err = -EINVAL;
5876                 goto out_unlock;
5877         }
5878
5879         if (attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR] &&
5880             attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]) {
5881                 if (!start_offset)
5882                         start_offset =
5883                                 nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
5884
5885                 end_offset = nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_ADDR]);
5886                 end_offset += nla_get_u64(attrs[DEVLINK_ATTR_REGION_CHUNK_LEN]);
5887         }
5888
5889         if (end_offset > region->size)
5890                 end_offset = region->size;
5891
5892         /* return 0 if there is no further data to read */
5893         if (start_offset == end_offset) {
5894                 err = 0;
5895                 goto out_unlock;
5896         }
5897
5898         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
5899                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI,
5900                           DEVLINK_CMD_REGION_READ);
5901         if (!hdr) {
5902                 err = -EMSGSIZE;
5903                 goto out_unlock;
5904         }
5905
5906         err = devlink_nl_put_handle(skb, devlink);
5907         if (err)
5908                 goto nla_put_failure;
5909
5910         if (region->port) {
5911                 err = nla_put_u32(skb, DEVLINK_ATTR_PORT_INDEX,
5912                                   region->port->index);
5913                 if (err)
5914                         goto nla_put_failure;
5915         }
5916
5917         err = nla_put_string(skb, DEVLINK_ATTR_REGION_NAME, region_name);
5918         if (err)
5919                 goto nla_put_failure;
5920
5921         chunks_attr = nla_nest_start_noflag(skb, DEVLINK_ATTR_REGION_CHUNKS);
5922         if (!chunks_attr) {
5923                 err = -EMSGSIZE;
5924                 goto nla_put_failure;
5925         }
5926
5927         err = devlink_nl_region_read_snapshot_fill(skb, devlink,
5928                                                    region, attrs,
5929                                                    start_offset,
5930                                                    end_offset, &ret_offset);
5931
5932         if (err && err != -EMSGSIZE)
5933                 goto nla_put_failure;
5934
5935         /* Check if there was any progress done to prevent infinite loop */
5936         if (ret_offset == start_offset) {
5937                 err = -EINVAL;
5938                 goto nla_put_failure;
5939         }
5940
5941         *((u64 *)&cb->args[0]) = ret_offset;
5942
5943         nla_nest_end(skb, chunks_attr);
5944         genlmsg_end(skb, hdr);
5945         mutex_unlock(&devlink->lock);
5946         devlink_put(devlink);
5947         mutex_unlock(&devlink_mutex);
5948
5949         return skb->len;
5950
5951 nla_put_failure:
5952         genlmsg_cancel(skb, hdr);
5953 out_unlock:
5954         mutex_unlock(&devlink->lock);
5955         devlink_put(devlink);
5956 out_dev:
5957         mutex_unlock(&devlink_mutex);
5958         return err;
5959 }
5960
5961 struct devlink_info_req {
5962         struct sk_buff *msg;
5963 };
5964
5965 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
5966 {
5967         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
5968 }
5969 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
5970
5971 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
5972 {
5973         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
5974 }
5975 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
5976
5977 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
5978                                          const char *bsn)
5979 {
5980         return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
5981                               bsn);
5982 }
5983 EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
5984
5985 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
5986                                     const char *version_name,
5987                                     const char *version_value)
5988 {
5989         struct nlattr *nest;
5990         int err;
5991
5992         nest = nla_nest_start_noflag(req->msg, attr);
5993         if (!nest)
5994                 return -EMSGSIZE;
5995
5996         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_NAME,
5997                              version_name);
5998         if (err)
5999                 goto nla_put_failure;
6000
6001         err = nla_put_string(req->msg, DEVLINK_ATTR_INFO_VERSION_VALUE,
6002                              version_value);
6003         if (err)
6004                 goto nla_put_failure;
6005
6006         nla_nest_end(req->msg, nest);
6007
6008         return 0;
6009
6010 nla_put_failure:
6011         nla_nest_cancel(req->msg, nest);
6012         return err;
6013 }
6014
6015 int devlink_info_version_fixed_put(struct devlink_info_req *req,
6016                                    const char *version_name,
6017                                    const char *version_value)
6018 {
6019         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
6020                                         version_name, version_value);
6021 }
6022 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
6023
6024 int devlink_info_version_stored_put(struct devlink_info_req *req,
6025                                     const char *version_name,
6026                                     const char *version_value)
6027 {
6028         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
6029                                         version_name, version_value);
6030 }
6031 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
6032
6033 int devlink_info_version_running_put(struct devlink_info_req *req,
6034                                      const char *version_name,
6035                                      const char *version_value)
6036 {
6037         return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
6038                                         version_name, version_value);
6039 }
6040 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
6041
6042 static int
6043 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
6044                      enum devlink_command cmd, u32 portid,
6045                      u32 seq, int flags, struct netlink_ext_ack *extack)
6046 {
6047         struct devlink_info_req req;
6048         void *hdr;
6049         int err;
6050
6051         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6052         if (!hdr)
6053                 return -EMSGSIZE;
6054
6055         err = -EMSGSIZE;
6056         if (devlink_nl_put_handle(msg, devlink))
6057                 goto err_cancel_msg;
6058
6059         req.msg = msg;
6060         err = devlink->ops->info_get(devlink, &req, extack);
6061         if (err)
6062                 goto err_cancel_msg;
6063
6064         genlmsg_end(msg, hdr);
6065         return 0;
6066
6067 err_cancel_msg:
6068         genlmsg_cancel(msg, hdr);
6069         return err;
6070 }
6071
6072 static int devlink_nl_cmd_info_get_doit(struct sk_buff *skb,
6073                                         struct genl_info *info)
6074 {
6075         struct devlink *devlink = info->user_ptr[0];
6076         struct sk_buff *msg;
6077         int err;
6078
6079         if (!devlink->ops->info_get)
6080                 return -EOPNOTSUPP;
6081
6082         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
6083         if (!msg)
6084                 return -ENOMEM;
6085
6086         err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6087                                    info->snd_portid, info->snd_seq, 0,
6088                                    info->extack);
6089         if (err) {
6090                 nlmsg_free(msg);
6091                 return err;
6092         }
6093
6094         return genlmsg_reply(msg, info);
6095 }
6096
6097 static int devlink_nl_cmd_info_get_dumpit(struct sk_buff *msg,
6098                                           struct netlink_callback *cb)
6099 {
6100         struct devlink *devlink;
6101         int start = cb->args[0];
6102         unsigned long index;
6103         int idx = 0;
6104         int err = 0;
6105
6106         mutex_lock(&devlink_mutex);
6107         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
6108                 if (!devlink_try_get(devlink))
6109                         continue;
6110
6111                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
6112                         goto retry;
6113
6114                 if (idx < start || !devlink->ops->info_get)
6115                         goto inc;
6116
6117                 mutex_lock(&devlink->lock);
6118                 err = devlink_nl_info_fill(msg, devlink, DEVLINK_CMD_INFO_GET,
6119                                            NETLINK_CB(cb->skb).portid,
6120                                            cb->nlh->nlmsg_seq, NLM_F_MULTI,
6121                                            cb->extack);
6122                 mutex_unlock(&devlink->lock);
6123                 if (err == -EOPNOTSUPP)
6124                         err = 0;
6125                 else if (err) {
6126                         devlink_put(devlink);
6127                         break;
6128                 }
6129 inc:
6130                 idx++;
6131 retry:
6132                 devlink_put(devlink);
6133         }
6134         mutex_unlock(&devlink_mutex);
6135
6136         if (err != -EMSGSIZE)
6137                 return err;
6138
6139         cb->args[0] = idx;
6140         return msg->len;
6141 }
6142
6143 struct devlink_fmsg_item {
6144         struct list_head list;
6145         int attrtype;
6146         u8 nla_type;
6147         u16 len;
6148         int value[];
6149 };
6150
6151 struct devlink_fmsg {
6152         struct list_head item_list;
6153         bool putting_binary; /* This flag forces enclosing of binary data
6154                               * in an array brackets. It forces using
6155                               * of designated API:
6156                               * devlink_fmsg_binary_pair_nest_start()
6157                               * devlink_fmsg_binary_pair_nest_end()
6158                               */
6159 };
6160
6161 static struct devlink_fmsg *devlink_fmsg_alloc(void)
6162 {
6163         struct devlink_fmsg *fmsg;
6164
6165         fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
6166         if (!fmsg)
6167                 return NULL;
6168
6169         INIT_LIST_HEAD(&fmsg->item_list);
6170
6171         return fmsg;
6172 }
6173
6174 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
6175 {
6176         struct devlink_fmsg_item *item, *tmp;
6177
6178         list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
6179                 list_del(&item->list);
6180                 kfree(item);
6181         }
6182         kfree(fmsg);
6183 }
6184
6185 static int devlink_fmsg_nest_common(struct devlink_fmsg *fmsg,
6186                                     int attrtype)
6187 {
6188         struct devlink_fmsg_item *item;
6189
6190         item = kzalloc(sizeof(*item), GFP_KERNEL);
6191         if (!item)
6192                 return -ENOMEM;
6193
6194         item->attrtype = attrtype;
6195         list_add_tail(&item->list, &fmsg->item_list);
6196
6197         return 0;
6198 }
6199
6200 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
6201 {
6202         if (fmsg->putting_binary)
6203                 return -EINVAL;
6204
6205         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
6206 }
6207 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
6208
6209 static int devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
6210 {
6211         if (fmsg->putting_binary)
6212                 return -EINVAL;
6213
6214         return devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
6215 }
6216
6217 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
6218 {
6219         if (fmsg->putting_binary)
6220                 return -EINVAL;
6221
6222         return devlink_fmsg_nest_end(fmsg);
6223 }
6224 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
6225
6226 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
6227
6228 static int devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
6229 {
6230         struct devlink_fmsg_item *item;
6231
6232         if (fmsg->putting_binary)
6233                 return -EINVAL;
6234
6235         if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE)
6236                 return -EMSGSIZE;
6237
6238         item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
6239         if (!item)
6240                 return -ENOMEM;
6241
6242         item->nla_type = NLA_NUL_STRING;
6243         item->len = strlen(name) + 1;
6244         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
6245         memcpy(&item->value, name, item->len);
6246         list_add_tail(&item->list, &fmsg->item_list);
6247
6248         return 0;
6249 }
6250
6251 int devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
6252 {
6253         int err;
6254
6255         if (fmsg->putting_binary)
6256                 return -EINVAL;
6257
6258         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
6259         if (err)
6260                 return err;
6261
6262         err = devlink_fmsg_put_name(fmsg, name);
6263         if (err)
6264                 return err;
6265
6266         return 0;
6267 }
6268 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
6269
6270 int devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
6271 {
6272         if (fmsg->putting_binary)
6273                 return -EINVAL;
6274
6275         return devlink_fmsg_nest_end(fmsg);
6276 }
6277 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
6278
6279 int devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
6280                                      const char *name)
6281 {
6282         int err;
6283
6284         if (fmsg->putting_binary)
6285                 return -EINVAL;
6286
6287         err = devlink_fmsg_pair_nest_start(fmsg, name);
6288         if (err)
6289                 return err;
6290
6291         err = devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
6292         if (err)
6293                 return err;
6294
6295         return 0;
6296 }
6297 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
6298
6299 int devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
6300 {
6301         int err;
6302
6303         if (fmsg->putting_binary)
6304                 return -EINVAL;
6305
6306         err = devlink_fmsg_nest_end(fmsg);
6307         if (err)
6308                 return err;
6309
6310         err = devlink_fmsg_nest_end(fmsg);
6311         if (err)
6312                 return err;
6313
6314         return 0;
6315 }
6316 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
6317
6318 int devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
6319                                         const char *name)
6320 {
6321         int err;
6322
6323         err = devlink_fmsg_arr_pair_nest_start(fmsg, name);
6324         if (err)
6325                 return err;
6326
6327         fmsg->putting_binary = true;
6328         return err;
6329 }
6330 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
6331
6332 int devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
6333 {
6334         if (!fmsg->putting_binary)
6335                 return -EINVAL;
6336
6337         fmsg->putting_binary = false;
6338         return devlink_fmsg_arr_pair_nest_end(fmsg);
6339 }
6340 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
6341
6342 static int devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
6343                                   const void *value, u16 value_len,
6344                                   u8 value_nla_type)
6345 {
6346         struct devlink_fmsg_item *item;
6347
6348         if (value_len > DEVLINK_FMSG_MAX_SIZE)
6349                 return -EMSGSIZE;
6350
6351         item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
6352         if (!item)
6353                 return -ENOMEM;
6354
6355         item->nla_type = value_nla_type;
6356         item->len = value_len;
6357         item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6358         memcpy(&item->value, value, item->len);
6359         list_add_tail(&item->list, &fmsg->item_list);
6360
6361         return 0;
6362 }
6363
6364 static int devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
6365 {
6366         if (fmsg->putting_binary)
6367                 return -EINVAL;
6368
6369         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
6370 }
6371
6372 static int devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
6373 {
6374         if (fmsg->putting_binary)
6375                 return -EINVAL;
6376
6377         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
6378 }
6379
6380 int devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
6381 {
6382         if (fmsg->putting_binary)
6383                 return -EINVAL;
6384
6385         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
6386 }
6387 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
6388
6389 static int devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
6390 {
6391         if (fmsg->putting_binary)
6392                 return -EINVAL;
6393
6394         return devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
6395 }
6396
6397 int devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
6398 {
6399         if (fmsg->putting_binary)
6400                 return -EINVAL;
6401
6402         return devlink_fmsg_put_value(fmsg, value, strlen(value) + 1,
6403                                       NLA_NUL_STRING);
6404 }
6405 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
6406
6407 int devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
6408                             u16 value_len)
6409 {
6410         if (!fmsg->putting_binary)
6411                 return -EINVAL;
6412
6413         return devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
6414 }
6415 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
6416
6417 int devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
6418                                bool value)
6419 {
6420         int err;
6421
6422         err = devlink_fmsg_pair_nest_start(fmsg, name);
6423         if (err)
6424                 return err;
6425
6426         err = devlink_fmsg_bool_put(fmsg, value);
6427         if (err)
6428                 return err;
6429
6430         err = devlink_fmsg_pair_nest_end(fmsg);
6431         if (err)
6432                 return err;
6433
6434         return 0;
6435 }
6436 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
6437
6438 int devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
6439                              u8 value)
6440 {
6441         int err;
6442
6443         err = devlink_fmsg_pair_nest_start(fmsg, name);
6444         if (err)
6445                 return err;
6446
6447         err = devlink_fmsg_u8_put(fmsg, value);
6448         if (err)
6449                 return err;
6450
6451         err = devlink_fmsg_pair_nest_end(fmsg);
6452         if (err)
6453                 return err;
6454
6455         return 0;
6456 }
6457 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
6458
6459 int devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
6460                               u32 value)
6461 {
6462         int err;
6463
6464         err = devlink_fmsg_pair_nest_start(fmsg, name);
6465         if (err)
6466                 return err;
6467
6468         err = devlink_fmsg_u32_put(fmsg, value);
6469         if (err)
6470                 return err;
6471
6472         err = devlink_fmsg_pair_nest_end(fmsg);
6473         if (err)
6474                 return err;
6475
6476         return 0;
6477 }
6478 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
6479
6480 int devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
6481                               u64 value)
6482 {
6483         int err;
6484
6485         err = devlink_fmsg_pair_nest_start(fmsg, name);
6486         if (err)
6487                 return err;
6488
6489         err = devlink_fmsg_u64_put(fmsg, value);
6490         if (err)
6491                 return err;
6492
6493         err = devlink_fmsg_pair_nest_end(fmsg);
6494         if (err)
6495                 return err;
6496
6497         return 0;
6498 }
6499 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
6500
6501 int devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
6502                                  const char *value)
6503 {
6504         int err;
6505
6506         err = devlink_fmsg_pair_nest_start(fmsg, name);
6507         if (err)
6508                 return err;
6509
6510         err = devlink_fmsg_string_put(fmsg, value);
6511         if (err)
6512                 return err;
6513
6514         err = devlink_fmsg_pair_nest_end(fmsg);
6515         if (err)
6516                 return err;
6517
6518         return 0;
6519 }
6520 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
6521
6522 int devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
6523                                  const void *value, u32 value_len)
6524 {
6525         u32 data_size;
6526         int end_err;
6527         u32 offset;
6528         int err;
6529
6530         err = devlink_fmsg_binary_pair_nest_start(fmsg, name);
6531         if (err)
6532                 return err;
6533
6534         for (offset = 0; offset < value_len; offset += data_size) {
6535                 data_size = value_len - offset;
6536                 if (data_size > DEVLINK_FMSG_MAX_SIZE)
6537                         data_size = DEVLINK_FMSG_MAX_SIZE;
6538                 err = devlink_fmsg_binary_put(fmsg, value + offset, data_size);
6539                 if (err)
6540                         break;
6541                 /* Exit from loop with a break (instead of
6542                  * return) to make sure putting_binary is turned off in
6543                  * devlink_fmsg_binary_pair_nest_end
6544                  */
6545         }
6546
6547         end_err = devlink_fmsg_binary_pair_nest_end(fmsg);
6548         if (end_err)
6549                 err = end_err;
6550
6551         return err;
6552 }
6553 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
6554
6555 static int
6556 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6557 {
6558         switch (msg->nla_type) {
6559         case NLA_FLAG:
6560         case NLA_U8:
6561         case NLA_U32:
6562         case NLA_U64:
6563         case NLA_NUL_STRING:
6564         case NLA_BINARY:
6565                 return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
6566                                   msg->nla_type);
6567         default:
6568                 return -EINVAL;
6569         }
6570 }
6571
6572 static int
6573 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
6574 {
6575         int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
6576         u8 tmp;
6577
6578         switch (msg->nla_type) {
6579         case NLA_FLAG:
6580                 /* Always provide flag data, regardless of its value */
6581                 tmp = *(bool *) msg->value;
6582
6583                 return nla_put_u8(skb, attrtype, tmp);
6584         case NLA_U8:
6585                 return nla_put_u8(skb, attrtype, *(u8 *) msg->value);
6586         case NLA_U32:
6587                 return nla_put_u32(skb, attrtype, *(u32 *) msg->value);
6588         case NLA_U64:
6589                 return nla_put_u64_64bit(skb, attrtype, *(u64 *) msg->value,
6590                                          DEVLINK_ATTR_PAD);
6591         case NLA_NUL_STRING:
6592                 return nla_put_string(skb, attrtype, (char *) &msg->value);
6593         case NLA_BINARY:
6594                 return nla_put(skb, attrtype, msg->len, (void *) &msg->value);
6595         default:
6596                 return -EINVAL;
6597         }
6598 }
6599
6600 static int
6601 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
6602                          int *start)
6603 {
6604         struct devlink_fmsg_item *item;
6605         struct nlattr *fmsg_nlattr;
6606         int i = 0;
6607         int err;
6608
6609         fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
6610         if (!fmsg_nlattr)
6611                 return -EMSGSIZE;
6612
6613         list_for_each_entry(item, &fmsg->item_list, list) {
6614                 if (i < *start) {
6615                         i++;
6616                         continue;
6617                 }
6618
6619                 switch (item->attrtype) {
6620                 case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
6621                 case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
6622                 case DEVLINK_ATTR_FMSG_ARR_NEST_START:
6623                 case DEVLINK_ATTR_FMSG_NEST_END:
6624                         err = nla_put_flag(skb, item->attrtype);
6625                         break;
6626                 case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
6627                         err = devlink_fmsg_item_fill_type(item, skb);
6628                         if (err)
6629                                 break;
6630                         err = devlink_fmsg_item_fill_data(item, skb);
6631                         break;
6632                 case DEVLINK_ATTR_FMSG_OBJ_NAME:
6633                         err = nla_put_string(skb, item->attrtype,
6634                                              (char *) &item->value);
6635                         break;
6636                 default:
6637                         err = -EINVAL;
6638                         break;
6639                 }
6640                 if (!err)
6641                         *start = ++i;
6642                 else
6643                         break;
6644         }
6645
6646         nla_nest_end(skb, fmsg_nlattr);
6647         return err;
6648 }
6649
6650 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
6651                             struct genl_info *info,
6652                             enum devlink_command cmd, int flags)
6653 {
6654         struct nlmsghdr *nlh;
6655         struct sk_buff *skb;
6656         bool last = false;
6657         int index = 0;
6658         void *hdr;
6659         int err;
6660
6661         while (!last) {
6662                 int tmp_index = index;
6663
6664                 skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
6665                 if (!skb)
6666                         return -ENOMEM;
6667
6668                 hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
6669                                   &devlink_nl_family, flags | NLM_F_MULTI, cmd);
6670                 if (!hdr) {
6671                         err = -EMSGSIZE;
6672                         goto nla_put_failure;
6673                 }
6674
6675                 err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
6676                 if (!err)
6677                         last = true;
6678                 else if (err != -EMSGSIZE || tmp_index == index)
6679                         goto nla_put_failure;
6680
6681                 genlmsg_end(skb, hdr);
6682                 err = genlmsg_reply(skb, info);
6683                 if (err)
6684                         return err;
6685         }
6686
6687         skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
6688         if (!skb)
6689                 return -ENOMEM;
6690         nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
6691                         NLMSG_DONE, 0, flags | NLM_F_MULTI);
6692         if (!nlh) {
6693                 err = -EMSGSIZE;
6694                 goto nla_put_failure;
6695         }
6696
6697         return genlmsg_reply(skb, info);
6698
6699 nla_put_failure:
6700         nlmsg_free(skb);
6701         return err;
6702 }
6703
6704 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
6705                                struct netlink_callback *cb,
6706                                enum devlink_command cmd)
6707 {
6708         int index = cb->args[0];
6709         int tmp_index = index;
6710         void *hdr;
6711         int err;
6712
6713         hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
6714                           &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
6715         if (!hdr) {
6716                 err = -EMSGSIZE;
6717                 goto nla_put_failure;
6718         }
6719
6720         err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
6721         if ((err && err != -EMSGSIZE) || tmp_index == index)
6722                 goto nla_put_failure;
6723
6724         cb->args[0] = index;
6725         genlmsg_end(skb, hdr);
6726         return skb->len;
6727
6728 nla_put_failure:
6729         genlmsg_cancel(skb, hdr);
6730         return err;
6731 }
6732
6733 struct devlink_health_reporter {
6734         struct list_head list;
6735         void *priv;
6736         const struct devlink_health_reporter_ops *ops;
6737         struct devlink *devlink;
6738         struct devlink_port *devlink_port;
6739         struct devlink_fmsg *dump_fmsg;
6740         struct mutex dump_lock; /* lock parallel read/write from dump buffers */
6741         u64 graceful_period;
6742         bool auto_recover;
6743         bool auto_dump;
6744         u8 health_state;
6745         u64 dump_ts;
6746         u64 dump_real_ts;
6747         u64 error_count;
6748         u64 recovery_count;
6749         u64 last_recovery_ts;
6750         refcount_t refcount;
6751 };
6752
6753 void *
6754 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
6755 {
6756         return reporter->priv;
6757 }
6758 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
6759
6760 static struct devlink_health_reporter *
6761 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
6762                                        struct mutex *list_lock,
6763                                        const char *reporter_name)
6764 {
6765         struct devlink_health_reporter *reporter;
6766
6767         lockdep_assert_held(list_lock);
6768         list_for_each_entry(reporter, reporter_list, list)
6769                 if (!strcmp(reporter->ops->name, reporter_name))
6770                         return reporter;
6771         return NULL;
6772 }
6773
6774 static struct devlink_health_reporter *
6775 devlink_health_reporter_find_by_name(struct devlink *devlink,
6776                                      const char *reporter_name)
6777 {
6778         return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
6779                                                       &devlink->reporters_lock,
6780                                                       reporter_name);
6781 }
6782
6783 static struct devlink_health_reporter *
6784 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
6785                                           const char *reporter_name)
6786 {
6787         return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
6788                                                       &devlink_port->reporters_lock,
6789                                                       reporter_name);
6790 }
6791
6792 static struct devlink_health_reporter *
6793 __devlink_health_reporter_create(struct devlink *devlink,
6794                                  const struct devlink_health_reporter_ops *ops,
6795                                  u64 graceful_period, void *priv)
6796 {
6797         struct devlink_health_reporter *reporter;
6798
6799         if (WARN_ON(graceful_period && !ops->recover))
6800                 return ERR_PTR(-EINVAL);
6801
6802         reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
6803         if (!reporter)
6804                 return ERR_PTR(-ENOMEM);
6805
6806         reporter->priv = priv;
6807         reporter->ops = ops;
6808         reporter->devlink = devlink;
6809         reporter->graceful_period = graceful_period;
6810         reporter->auto_recover = !!ops->recover;
6811         reporter->auto_dump = !!ops->dump;
6812         mutex_init(&reporter->dump_lock);
6813         refcount_set(&reporter->refcount, 1);
6814         return reporter;
6815 }
6816
6817 /**
6818  *      devlink_port_health_reporter_create - create devlink health reporter for
6819  *                                            specified port instance
6820  *
6821  *      @port: devlink_port which should contain the new reporter
6822  *      @ops: ops
6823  *      @graceful_period: to avoid recovery loops, in msecs
6824  *      @priv: priv
6825  */
6826 struct devlink_health_reporter *
6827 devlink_port_health_reporter_create(struct devlink_port *port,
6828                                     const struct devlink_health_reporter_ops *ops,
6829                                     u64 graceful_period, void *priv)
6830 {
6831         struct devlink_health_reporter *reporter;
6832
6833         mutex_lock(&port->reporters_lock);
6834         if (__devlink_health_reporter_find_by_name(&port->reporter_list,
6835                                                    &port->reporters_lock, ops->name)) {
6836                 reporter = ERR_PTR(-EEXIST);
6837                 goto unlock;
6838         }
6839
6840         reporter = __devlink_health_reporter_create(port->devlink, ops,
6841                                                     graceful_period, priv);
6842         if (IS_ERR(reporter))
6843                 goto unlock;
6844
6845         reporter->devlink_port = port;
6846         list_add_tail(&reporter->list, &port->reporter_list);
6847 unlock:
6848         mutex_unlock(&port->reporters_lock);
6849         return reporter;
6850 }
6851 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
6852
6853 /**
6854  *      devlink_health_reporter_create - create devlink health reporter
6855  *
6856  *      @devlink: devlink
6857  *      @ops: ops
6858  *      @graceful_period: to avoid recovery loops, in msecs
6859  *      @priv: priv
6860  */
6861 struct devlink_health_reporter *
6862 devlink_health_reporter_create(struct devlink *devlink,
6863                                const struct devlink_health_reporter_ops *ops,
6864                                u64 graceful_period, void *priv)
6865 {
6866         struct devlink_health_reporter *reporter;
6867
6868         mutex_lock(&devlink->reporters_lock);
6869         if (devlink_health_reporter_find_by_name(devlink, ops->name)) {
6870                 reporter = ERR_PTR(-EEXIST);
6871                 goto unlock;
6872         }
6873
6874         reporter = __devlink_health_reporter_create(devlink, ops,
6875                                                     graceful_period, priv);
6876         if (IS_ERR(reporter))
6877                 goto unlock;
6878
6879         list_add_tail(&reporter->list, &devlink->reporter_list);
6880 unlock:
6881         mutex_unlock(&devlink->reporters_lock);
6882         return reporter;
6883 }
6884 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
6885
6886 static void
6887 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
6888 {
6889         mutex_destroy(&reporter->dump_lock);
6890         if (reporter->dump_fmsg)
6891                 devlink_fmsg_free(reporter->dump_fmsg);
6892         kfree(reporter);
6893 }
6894
6895 static void
6896 devlink_health_reporter_put(struct devlink_health_reporter *reporter)
6897 {
6898         if (refcount_dec_and_test(&reporter->refcount))
6899                 devlink_health_reporter_free(reporter);
6900 }
6901
6902 static void
6903 __devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
6904 {
6905         list_del(&reporter->list);
6906         devlink_health_reporter_put(reporter);
6907 }
6908
6909 /**
6910  *      devlink_health_reporter_destroy - destroy devlink health reporter
6911  *
6912  *      @reporter: devlink health reporter to destroy
6913  */
6914 void
6915 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
6916 {
6917         struct mutex *lock = &reporter->devlink->reporters_lock;
6918
6919         mutex_lock(lock);
6920         __devlink_health_reporter_destroy(reporter);
6921         mutex_unlock(lock);
6922 }
6923 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
6924
6925 /**
6926  *      devlink_port_health_reporter_destroy - destroy devlink port health reporter
6927  *
6928  *      @reporter: devlink health reporter to destroy
6929  */
6930 void
6931 devlink_port_health_reporter_destroy(struct devlink_health_reporter *reporter)
6932 {
6933         struct mutex *lock = &reporter->devlink_port->reporters_lock;
6934
6935         mutex_lock(lock);
6936         __devlink_health_reporter_destroy(reporter);
6937         mutex_unlock(lock);
6938 }
6939 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_destroy);
6940
6941 static int
6942 devlink_nl_health_reporter_fill(struct sk_buff *msg,
6943                                 struct devlink_health_reporter *reporter,
6944                                 enum devlink_command cmd, u32 portid,
6945                                 u32 seq, int flags)
6946 {
6947         struct devlink *devlink = reporter->devlink;
6948         struct nlattr *reporter_attr;
6949         void *hdr;
6950
6951         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
6952         if (!hdr)
6953                 return -EMSGSIZE;
6954
6955         if (devlink_nl_put_handle(msg, devlink))
6956                 goto genlmsg_cancel;
6957
6958         if (reporter->devlink_port) {
6959                 if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
6960                         goto genlmsg_cancel;
6961         }
6962         reporter_attr = nla_nest_start_noflag(msg,
6963                                               DEVLINK_ATTR_HEALTH_REPORTER);
6964         if (!reporter_attr)
6965                 goto genlmsg_cancel;
6966         if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
6967                            reporter->ops->name))
6968                 goto reporter_nest_cancel;
6969         if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
6970                        reporter->health_state))
6971                 goto reporter_nest_cancel;
6972         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
6973                               reporter->error_count, DEVLINK_ATTR_PAD))
6974                 goto reporter_nest_cancel;
6975         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
6976                               reporter->recovery_count, DEVLINK_ATTR_PAD))
6977                 goto reporter_nest_cancel;
6978         if (reporter->ops->recover &&
6979             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
6980                               reporter->graceful_period,
6981                               DEVLINK_ATTR_PAD))
6982                 goto reporter_nest_cancel;
6983         if (reporter->ops->recover &&
6984             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
6985                        reporter->auto_recover))
6986                 goto reporter_nest_cancel;
6987         if (reporter->dump_fmsg &&
6988             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
6989                               jiffies_to_msecs(reporter->dump_ts),
6990                               DEVLINK_ATTR_PAD))
6991                 goto reporter_nest_cancel;
6992         if (reporter->dump_fmsg &&
6993             nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
6994                               reporter->dump_real_ts, DEVLINK_ATTR_PAD))
6995                 goto reporter_nest_cancel;
6996         if (reporter->ops->dump &&
6997             nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
6998                        reporter->auto_dump))
6999                 goto reporter_nest_cancel;
7000
7001         nla_nest_end(msg, reporter_attr);
7002         genlmsg_end(msg, hdr);
7003         return 0;
7004
7005 reporter_nest_cancel:
7006         nla_nest_end(msg, reporter_attr);
7007 genlmsg_cancel:
7008         genlmsg_cancel(msg, hdr);
7009         return -EMSGSIZE;
7010 }
7011
7012 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
7013                                    enum devlink_command cmd)
7014 {
7015         struct devlink *devlink = reporter->devlink;
7016         struct sk_buff *msg;
7017         int err;
7018
7019         WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7020         WARN_ON(!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED));
7021
7022         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7023         if (!msg)
7024                 return;
7025
7026         err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
7027         if (err) {
7028                 nlmsg_free(msg);
7029                 return;
7030         }
7031
7032         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
7033                                 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
7034 }
7035
7036 void
7037 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
7038 {
7039         reporter->recovery_count++;
7040         reporter->last_recovery_ts = jiffies;
7041 }
7042 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
7043
7044 static int
7045 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
7046                                 void *priv_ctx, struct netlink_ext_ack *extack)
7047 {
7048         int err;
7049
7050         if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
7051                 return 0;
7052
7053         if (!reporter->ops->recover)
7054                 return -EOPNOTSUPP;
7055
7056         err = reporter->ops->recover(reporter, priv_ctx, extack);
7057         if (err)
7058                 return err;
7059
7060         devlink_health_reporter_recovery_done(reporter);
7061         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
7062         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7063
7064         return 0;
7065 }
7066
7067 static void
7068 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
7069 {
7070         if (!reporter->dump_fmsg)
7071                 return;
7072         devlink_fmsg_free(reporter->dump_fmsg);
7073         reporter->dump_fmsg = NULL;
7074 }
7075
7076 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
7077                                   void *priv_ctx,
7078                                   struct netlink_ext_ack *extack)
7079 {
7080         int err;
7081
7082         if (!reporter->ops->dump)
7083                 return 0;
7084
7085         if (reporter->dump_fmsg)
7086                 return 0;
7087
7088         reporter->dump_fmsg = devlink_fmsg_alloc();
7089         if (!reporter->dump_fmsg) {
7090                 err = -ENOMEM;
7091                 return err;
7092         }
7093
7094         err = devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
7095         if (err)
7096                 goto dump_err;
7097
7098         err = reporter->ops->dump(reporter, reporter->dump_fmsg,
7099                                   priv_ctx, extack);
7100         if (err)
7101                 goto dump_err;
7102
7103         err = devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
7104         if (err)
7105                 goto dump_err;
7106
7107         reporter->dump_ts = jiffies;
7108         reporter->dump_real_ts = ktime_get_real_ns();
7109
7110         return 0;
7111
7112 dump_err:
7113         devlink_health_dump_clear(reporter);
7114         return err;
7115 }
7116
7117 int devlink_health_report(struct devlink_health_reporter *reporter,
7118                           const char *msg, void *priv_ctx)
7119 {
7120         enum devlink_health_reporter_state prev_health_state;
7121         struct devlink *devlink = reporter->devlink;
7122         unsigned long recover_ts_threshold;
7123
7124         /* write a log message of the current error */
7125         WARN_ON(!msg);
7126         trace_devlink_health_report(devlink, reporter->ops->name, msg);
7127         reporter->error_count++;
7128         prev_health_state = reporter->health_state;
7129         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7130         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7131
7132         /* abort if the previous error wasn't recovered */
7133         recover_ts_threshold = reporter->last_recovery_ts +
7134                                msecs_to_jiffies(reporter->graceful_period);
7135         if (reporter->auto_recover &&
7136             (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
7137              (reporter->last_recovery_ts && reporter->recovery_count &&
7138               time_is_after_jiffies(recover_ts_threshold)))) {
7139                 trace_devlink_health_recover_aborted(devlink,
7140                                                      reporter->ops->name,
7141                                                      reporter->health_state,
7142                                                      jiffies -
7143                                                      reporter->last_recovery_ts);
7144                 return -ECANCELED;
7145         }
7146
7147         reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
7148
7149         if (reporter->auto_dump) {
7150                 mutex_lock(&reporter->dump_lock);
7151                 /* store current dump of current error, for later analysis */
7152                 devlink_health_do_dump(reporter, priv_ctx, NULL);
7153                 mutex_unlock(&reporter->dump_lock);
7154         }
7155
7156         if (reporter->auto_recover)
7157                 return devlink_health_reporter_recover(reporter,
7158                                                        priv_ctx, NULL);
7159
7160         return 0;
7161 }
7162 EXPORT_SYMBOL_GPL(devlink_health_report);
7163
7164 static struct devlink_health_reporter *
7165 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
7166                                        struct nlattr **attrs)
7167 {
7168         struct devlink_health_reporter *reporter;
7169         struct devlink_port *devlink_port;
7170         char *reporter_name;
7171
7172         if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
7173                 return NULL;
7174
7175         reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
7176         devlink_port = devlink_port_get_from_attrs(devlink, attrs);
7177         if (IS_ERR(devlink_port)) {
7178                 mutex_lock(&devlink->reporters_lock);
7179                 reporter = devlink_health_reporter_find_by_name(devlink, reporter_name);
7180                 if (reporter)
7181                         refcount_inc(&reporter->refcount);
7182                 mutex_unlock(&devlink->reporters_lock);
7183         } else {
7184                 mutex_lock(&devlink_port->reporters_lock);
7185                 reporter = devlink_port_health_reporter_find_by_name(devlink_port, reporter_name);
7186                 if (reporter)
7187                         refcount_inc(&reporter->refcount);
7188                 mutex_unlock(&devlink_port->reporters_lock);
7189         }
7190
7191         return reporter;
7192 }
7193
7194 static struct devlink_health_reporter *
7195 devlink_health_reporter_get_from_info(struct devlink *devlink,
7196                                       struct genl_info *info)
7197 {
7198         return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
7199 }
7200
7201 static struct devlink_health_reporter *
7202 devlink_health_reporter_get_from_cb(struct netlink_callback *cb)
7203 {
7204         const struct genl_dumpit_info *info = genl_dumpit_info(cb);
7205         struct devlink_health_reporter *reporter;
7206         struct nlattr **attrs = info->attrs;
7207         struct devlink *devlink;
7208
7209         mutex_lock(&devlink_mutex);
7210         devlink = devlink_get_from_attrs(sock_net(cb->skb->sk), attrs);
7211         if (IS_ERR(devlink))
7212                 goto unlock;
7213
7214         reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
7215         devlink_put(devlink);
7216         mutex_unlock(&devlink_mutex);
7217         return reporter;
7218 unlock:
7219         mutex_unlock(&devlink_mutex);
7220         return NULL;
7221 }
7222
7223 void
7224 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
7225                                      enum devlink_health_reporter_state state)
7226 {
7227         if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
7228                     state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
7229                 return;
7230
7231         if (reporter->health_state == state)
7232                 return;
7233
7234         reporter->health_state = state;
7235         trace_devlink_health_reporter_state_update(reporter->devlink,
7236                                                    reporter->ops->name, state);
7237         devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
7238 }
7239 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
7240
7241 static int devlink_nl_cmd_health_reporter_get_doit(struct sk_buff *skb,
7242                                                    struct genl_info *info)
7243 {
7244         struct devlink *devlink = info->user_ptr[0];
7245         struct devlink_health_reporter *reporter;
7246         struct sk_buff *msg;
7247         int err;
7248
7249         reporter = devlink_health_reporter_get_from_info(devlink, info);
7250         if (!reporter)
7251                 return -EINVAL;
7252
7253         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7254         if (!msg) {
7255                 err = -ENOMEM;
7256                 goto out;
7257         }
7258
7259         err = devlink_nl_health_reporter_fill(msg, reporter,
7260                                               DEVLINK_CMD_HEALTH_REPORTER_GET,
7261                                               info->snd_portid, info->snd_seq,
7262                                               0);
7263         if (err) {
7264                 nlmsg_free(msg);
7265                 goto out;
7266         }
7267
7268         err = genlmsg_reply(msg, info);
7269 out:
7270         devlink_health_reporter_put(reporter);
7271         return err;
7272 }
7273
7274 static int
7275 devlink_nl_cmd_health_reporter_get_dumpit(struct sk_buff *msg,
7276                                           struct netlink_callback *cb)
7277 {
7278         struct devlink_health_reporter *reporter;
7279         struct devlink_port *port;
7280         struct devlink *devlink;
7281         int start = cb->args[0];
7282         unsigned long index;
7283         int idx = 0;
7284         int err;
7285
7286         mutex_lock(&devlink_mutex);
7287         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7288                 if (!devlink_try_get(devlink))
7289                         continue;
7290
7291                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7292                         goto retry_rep;
7293
7294                 mutex_lock(&devlink->reporters_lock);
7295                 list_for_each_entry(reporter, &devlink->reporter_list,
7296                                     list) {
7297                         if (idx < start) {
7298                                 idx++;
7299                                 continue;
7300                         }
7301                         err = devlink_nl_health_reporter_fill(
7302                                 msg, reporter, DEVLINK_CMD_HEALTH_REPORTER_GET,
7303                                 NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
7304                                 NLM_F_MULTI);
7305                         if (err) {
7306                                 mutex_unlock(&devlink->reporters_lock);
7307                                 devlink_put(devlink);
7308                                 goto out;
7309                         }
7310                         idx++;
7311                 }
7312                 mutex_unlock(&devlink->reporters_lock);
7313 retry_rep:
7314                 devlink_put(devlink);
7315         }
7316
7317         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7318                 if (!devlink_try_get(devlink))
7319                         continue;
7320
7321                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7322                         goto retry_port;
7323
7324                 mutex_lock(&devlink->lock);
7325                 list_for_each_entry(port, &devlink->port_list, list) {
7326                         mutex_lock(&port->reporters_lock);
7327                         list_for_each_entry(reporter, &port->reporter_list, list) {
7328                                 if (idx < start) {
7329                                         idx++;
7330                                         continue;
7331                                 }
7332                                 err = devlink_nl_health_reporter_fill(
7333                                         msg, reporter,
7334                                         DEVLINK_CMD_HEALTH_REPORTER_GET,
7335                                         NETLINK_CB(cb->skb).portid,
7336                                         cb->nlh->nlmsg_seq, NLM_F_MULTI);
7337                                 if (err) {
7338                                         mutex_unlock(&port->reporters_lock);
7339                                         mutex_unlock(&devlink->lock);
7340                                         devlink_put(devlink);
7341                                         goto out;
7342                                 }
7343                                 idx++;
7344                         }
7345                         mutex_unlock(&port->reporters_lock);
7346                 }
7347                 mutex_unlock(&devlink->lock);
7348 retry_port:
7349                 devlink_put(devlink);
7350         }
7351 out:
7352         mutex_unlock(&devlink_mutex);
7353
7354         cb->args[0] = idx;
7355         return msg->len;
7356 }
7357
7358 static int
7359 devlink_nl_cmd_health_reporter_set_doit(struct sk_buff *skb,
7360                                         struct genl_info *info)
7361 {
7362         struct devlink *devlink = info->user_ptr[0];
7363         struct devlink_health_reporter *reporter;
7364         int err;
7365
7366         reporter = devlink_health_reporter_get_from_info(devlink, info);
7367         if (!reporter)
7368                 return -EINVAL;
7369
7370         if (!reporter->ops->recover &&
7371             (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
7372              info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])) {
7373                 err = -EOPNOTSUPP;
7374                 goto out;
7375         }
7376         if (!reporter->ops->dump &&
7377             info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]) {
7378                 err = -EOPNOTSUPP;
7379                 goto out;
7380         }
7381
7382         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
7383                 reporter->graceful_period =
7384                         nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
7385
7386         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
7387                 reporter->auto_recover =
7388                         nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
7389
7390         if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
7391                 reporter->auto_dump =
7392                 nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
7393
7394         devlink_health_reporter_put(reporter);
7395         return 0;
7396 out:
7397         devlink_health_reporter_put(reporter);
7398         return err;
7399 }
7400
7401 static int devlink_nl_cmd_health_reporter_recover_doit(struct sk_buff *skb,
7402                                                        struct genl_info *info)
7403 {
7404         struct devlink *devlink = info->user_ptr[0];
7405         struct devlink_health_reporter *reporter;
7406         int err;
7407
7408         reporter = devlink_health_reporter_get_from_info(devlink, info);
7409         if (!reporter)
7410                 return -EINVAL;
7411
7412         err = devlink_health_reporter_recover(reporter, NULL, info->extack);
7413
7414         devlink_health_reporter_put(reporter);
7415         return err;
7416 }
7417
7418 static int devlink_nl_cmd_health_reporter_diagnose_doit(struct sk_buff *skb,
7419                                                         struct genl_info *info)
7420 {
7421         struct devlink *devlink = info->user_ptr[0];
7422         struct devlink_health_reporter *reporter;
7423         struct devlink_fmsg *fmsg;
7424         int err;
7425
7426         reporter = devlink_health_reporter_get_from_info(devlink, info);
7427         if (!reporter)
7428                 return -EINVAL;
7429
7430         if (!reporter->ops->diagnose) {
7431                 devlink_health_reporter_put(reporter);
7432                 return -EOPNOTSUPP;
7433         }
7434
7435         fmsg = devlink_fmsg_alloc();
7436         if (!fmsg) {
7437                 devlink_health_reporter_put(reporter);
7438                 return -ENOMEM;
7439         }
7440
7441         err = devlink_fmsg_obj_nest_start(fmsg);
7442         if (err)
7443                 goto out;
7444
7445         err = reporter->ops->diagnose(reporter, fmsg, info->extack);
7446         if (err)
7447                 goto out;
7448
7449         err = devlink_fmsg_obj_nest_end(fmsg);
7450         if (err)
7451                 goto out;
7452
7453         err = devlink_fmsg_snd(fmsg, info,
7454                                DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
7455
7456 out:
7457         devlink_fmsg_free(fmsg);
7458         devlink_health_reporter_put(reporter);
7459         return err;
7460 }
7461
7462 static int
7463 devlink_nl_cmd_health_reporter_dump_get_dumpit(struct sk_buff *skb,
7464                                                struct netlink_callback *cb)
7465 {
7466         struct devlink_health_reporter *reporter;
7467         u64 start = cb->args[0];
7468         int err;
7469
7470         reporter = devlink_health_reporter_get_from_cb(cb);
7471         if (!reporter)
7472                 return -EINVAL;
7473
7474         if (!reporter->ops->dump) {
7475                 err = -EOPNOTSUPP;
7476                 goto out;
7477         }
7478         mutex_lock(&reporter->dump_lock);
7479         if (!start) {
7480                 err = devlink_health_do_dump(reporter, NULL, cb->extack);
7481                 if (err)
7482                         goto unlock;
7483                 cb->args[1] = reporter->dump_ts;
7484         }
7485         if (!reporter->dump_fmsg || cb->args[1] != reporter->dump_ts) {
7486                 NL_SET_ERR_MSG_MOD(cb->extack, "Dump trampled, please retry");
7487                 err = -EAGAIN;
7488                 goto unlock;
7489         }
7490
7491         err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
7492                                   DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
7493 unlock:
7494         mutex_unlock(&reporter->dump_lock);
7495 out:
7496         devlink_health_reporter_put(reporter);
7497         return err;
7498 }
7499
7500 static int
7501 devlink_nl_cmd_health_reporter_dump_clear_doit(struct sk_buff *skb,
7502                                                struct genl_info *info)
7503 {
7504         struct devlink *devlink = info->user_ptr[0];
7505         struct devlink_health_reporter *reporter;
7506
7507         reporter = devlink_health_reporter_get_from_info(devlink, info);
7508         if (!reporter)
7509                 return -EINVAL;
7510
7511         if (!reporter->ops->dump) {
7512                 devlink_health_reporter_put(reporter);
7513                 return -EOPNOTSUPP;
7514         }
7515
7516         mutex_lock(&reporter->dump_lock);
7517         devlink_health_dump_clear(reporter);
7518         mutex_unlock(&reporter->dump_lock);
7519         devlink_health_reporter_put(reporter);
7520         return 0;
7521 }
7522
7523 static int devlink_nl_cmd_health_reporter_test_doit(struct sk_buff *skb,
7524                                                     struct genl_info *info)
7525 {
7526         struct devlink *devlink = info->user_ptr[0];
7527         struct devlink_health_reporter *reporter;
7528         int err;
7529
7530         reporter = devlink_health_reporter_get_from_info(devlink, info);
7531         if (!reporter)
7532                 return -EINVAL;
7533
7534         if (!reporter->ops->test) {
7535                 devlink_health_reporter_put(reporter);
7536                 return -EOPNOTSUPP;
7537         }
7538
7539         err = reporter->ops->test(reporter, info->extack);
7540
7541         devlink_health_reporter_put(reporter);
7542         return err;
7543 }
7544
7545 struct devlink_stats {
7546         u64 rx_bytes;
7547         u64 rx_packets;
7548         struct u64_stats_sync syncp;
7549 };
7550
7551 /**
7552  * struct devlink_trap_policer_item - Packet trap policer attributes.
7553  * @policer: Immutable packet trap policer attributes.
7554  * @rate: Rate in packets / sec.
7555  * @burst: Burst size in packets.
7556  * @list: trap_policer_list member.
7557  *
7558  * Describes packet trap policer attributes. Created by devlink during trap
7559  * policer registration.
7560  */
7561 struct devlink_trap_policer_item {
7562         const struct devlink_trap_policer *policer;
7563         u64 rate;
7564         u64 burst;
7565         struct list_head list;
7566 };
7567
7568 /**
7569  * struct devlink_trap_group_item - Packet trap group attributes.
7570  * @group: Immutable packet trap group attributes.
7571  * @policer_item: Associated policer item. Can be NULL.
7572  * @list: trap_group_list member.
7573  * @stats: Trap group statistics.
7574  *
7575  * Describes packet trap group attributes. Created by devlink during trap
7576  * group registration.
7577  */
7578 struct devlink_trap_group_item {
7579         const struct devlink_trap_group *group;
7580         struct devlink_trap_policer_item *policer_item;
7581         struct list_head list;
7582         struct devlink_stats __percpu *stats;
7583 };
7584
7585 /**
7586  * struct devlink_trap_item - Packet trap attributes.
7587  * @trap: Immutable packet trap attributes.
7588  * @group_item: Associated group item.
7589  * @list: trap_list member.
7590  * @action: Trap action.
7591  * @stats: Trap statistics.
7592  * @priv: Driver private information.
7593  *
7594  * Describes both mutable and immutable packet trap attributes. Created by
7595  * devlink during trap registration and used for all trap related operations.
7596  */
7597 struct devlink_trap_item {
7598         const struct devlink_trap *trap;
7599         struct devlink_trap_group_item *group_item;
7600         struct list_head list;
7601         enum devlink_trap_action action;
7602         struct devlink_stats __percpu *stats;
7603         void *priv;
7604 };
7605
7606 static struct devlink_trap_policer_item *
7607 devlink_trap_policer_item_lookup(struct devlink *devlink, u32 id)
7608 {
7609         struct devlink_trap_policer_item *policer_item;
7610
7611         list_for_each_entry(policer_item, &devlink->trap_policer_list, list) {
7612                 if (policer_item->policer->id == id)
7613                         return policer_item;
7614         }
7615
7616         return NULL;
7617 }
7618
7619 static struct devlink_trap_item *
7620 devlink_trap_item_lookup(struct devlink *devlink, const char *name)
7621 {
7622         struct devlink_trap_item *trap_item;
7623
7624         list_for_each_entry(trap_item, &devlink->trap_list, list) {
7625                 if (!strcmp(trap_item->trap->name, name))
7626                         return trap_item;
7627         }
7628
7629         return NULL;
7630 }
7631
7632 static struct devlink_trap_item *
7633 devlink_trap_item_get_from_info(struct devlink *devlink,
7634                                 struct genl_info *info)
7635 {
7636         struct nlattr *attr;
7637
7638         if (!info->attrs[DEVLINK_ATTR_TRAP_NAME])
7639                 return NULL;
7640         attr = info->attrs[DEVLINK_ATTR_TRAP_NAME];
7641
7642         return devlink_trap_item_lookup(devlink, nla_data(attr));
7643 }
7644
7645 static int
7646 devlink_trap_action_get_from_info(struct genl_info *info,
7647                                   enum devlink_trap_action *p_trap_action)
7648 {
7649         u8 val;
7650
7651         val = nla_get_u8(info->attrs[DEVLINK_ATTR_TRAP_ACTION]);
7652         switch (val) {
7653         case DEVLINK_TRAP_ACTION_DROP:
7654         case DEVLINK_TRAP_ACTION_TRAP:
7655         case DEVLINK_TRAP_ACTION_MIRROR:
7656                 *p_trap_action = val;
7657                 break;
7658         default:
7659                 return -EINVAL;
7660         }
7661
7662         return 0;
7663 }
7664
7665 static int devlink_trap_metadata_put(struct sk_buff *msg,
7666                                      const struct devlink_trap *trap)
7667 {
7668         struct nlattr *attr;
7669
7670         attr = nla_nest_start(msg, DEVLINK_ATTR_TRAP_METADATA);
7671         if (!attr)
7672                 return -EMSGSIZE;
7673
7674         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_IN_PORT) &&
7675             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_IN_PORT))
7676                 goto nla_put_failure;
7677         if ((trap->metadata_cap & DEVLINK_TRAP_METADATA_TYPE_F_FA_COOKIE) &&
7678             nla_put_flag(msg, DEVLINK_ATTR_TRAP_METADATA_TYPE_FA_COOKIE))
7679                 goto nla_put_failure;
7680
7681         nla_nest_end(msg, attr);
7682
7683         return 0;
7684
7685 nla_put_failure:
7686         nla_nest_cancel(msg, attr);
7687         return -EMSGSIZE;
7688 }
7689
7690 static void devlink_trap_stats_read(struct devlink_stats __percpu *trap_stats,
7691                                     struct devlink_stats *stats)
7692 {
7693         int i;
7694
7695         memset(stats, 0, sizeof(*stats));
7696         for_each_possible_cpu(i) {
7697                 struct devlink_stats *cpu_stats;
7698                 u64 rx_packets, rx_bytes;
7699                 unsigned int start;
7700
7701                 cpu_stats = per_cpu_ptr(trap_stats, i);
7702                 do {
7703                         start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
7704                         rx_packets = cpu_stats->rx_packets;
7705                         rx_bytes = cpu_stats->rx_bytes;
7706                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
7707
7708                 stats->rx_packets += rx_packets;
7709                 stats->rx_bytes += rx_bytes;
7710         }
7711 }
7712
7713 static int
7714 devlink_trap_group_stats_put(struct sk_buff *msg,
7715                              struct devlink_stats __percpu *trap_stats)
7716 {
7717         struct devlink_stats stats;
7718         struct nlattr *attr;
7719
7720         devlink_trap_stats_read(trap_stats, &stats);
7721
7722         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
7723         if (!attr)
7724                 return -EMSGSIZE;
7725
7726         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
7727                               stats.rx_packets, DEVLINK_ATTR_PAD))
7728                 goto nla_put_failure;
7729
7730         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
7731                               stats.rx_bytes, DEVLINK_ATTR_PAD))
7732                 goto nla_put_failure;
7733
7734         nla_nest_end(msg, attr);
7735
7736         return 0;
7737
7738 nla_put_failure:
7739         nla_nest_cancel(msg, attr);
7740         return -EMSGSIZE;
7741 }
7742
7743 static int devlink_trap_stats_put(struct sk_buff *msg, struct devlink *devlink,
7744                                   const struct devlink_trap_item *trap_item)
7745 {
7746         struct devlink_stats stats;
7747         struct nlattr *attr;
7748         u64 drops = 0;
7749         int err;
7750
7751         if (devlink->ops->trap_drop_counter_get) {
7752                 err = devlink->ops->trap_drop_counter_get(devlink,
7753                                                           trap_item->trap,
7754                                                           &drops);
7755                 if (err)
7756                         return err;
7757         }
7758
7759         devlink_trap_stats_read(trap_item->stats, &stats);
7760
7761         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
7762         if (!attr)
7763                 return -EMSGSIZE;
7764
7765         if (devlink->ops->trap_drop_counter_get &&
7766             nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
7767                               DEVLINK_ATTR_PAD))
7768                 goto nla_put_failure;
7769
7770         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_PACKETS,
7771                               stats.rx_packets, DEVLINK_ATTR_PAD))
7772                 goto nla_put_failure;
7773
7774         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_BYTES,
7775                               stats.rx_bytes, DEVLINK_ATTR_PAD))
7776                 goto nla_put_failure;
7777
7778         nla_nest_end(msg, attr);
7779
7780         return 0;
7781
7782 nla_put_failure:
7783         nla_nest_cancel(msg, attr);
7784         return -EMSGSIZE;
7785 }
7786
7787 static int devlink_nl_trap_fill(struct sk_buff *msg, struct devlink *devlink,
7788                                 const struct devlink_trap_item *trap_item,
7789                                 enum devlink_command cmd, u32 portid, u32 seq,
7790                                 int flags)
7791 {
7792         struct devlink_trap_group_item *group_item = trap_item->group_item;
7793         void *hdr;
7794         int err;
7795
7796         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
7797         if (!hdr)
7798                 return -EMSGSIZE;
7799
7800         if (devlink_nl_put_handle(msg, devlink))
7801                 goto nla_put_failure;
7802
7803         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
7804                            group_item->group->name))
7805                 goto nla_put_failure;
7806
7807         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_NAME, trap_item->trap->name))
7808                 goto nla_put_failure;
7809
7810         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_TYPE, trap_item->trap->type))
7811                 goto nla_put_failure;
7812
7813         if (trap_item->trap->generic &&
7814             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
7815                 goto nla_put_failure;
7816
7817         if (nla_put_u8(msg, DEVLINK_ATTR_TRAP_ACTION, trap_item->action))
7818                 goto nla_put_failure;
7819
7820         err = devlink_trap_metadata_put(msg, trap_item->trap);
7821         if (err)
7822                 goto nla_put_failure;
7823
7824         err = devlink_trap_stats_put(msg, devlink, trap_item);
7825         if (err)
7826                 goto nla_put_failure;
7827
7828         genlmsg_end(msg, hdr);
7829
7830         return 0;
7831
7832 nla_put_failure:
7833         genlmsg_cancel(msg, hdr);
7834         return -EMSGSIZE;
7835 }
7836
7837 static int devlink_nl_cmd_trap_get_doit(struct sk_buff *skb,
7838                                         struct genl_info *info)
7839 {
7840         struct netlink_ext_ack *extack = info->extack;
7841         struct devlink *devlink = info->user_ptr[0];
7842         struct devlink_trap_item *trap_item;
7843         struct sk_buff *msg;
7844         int err;
7845
7846         if (list_empty(&devlink->trap_list))
7847                 return -EOPNOTSUPP;
7848
7849         trap_item = devlink_trap_item_get_from_info(devlink, info);
7850         if (!trap_item) {
7851                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
7852                 return -ENOENT;
7853         }
7854
7855         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
7856         if (!msg)
7857                 return -ENOMEM;
7858
7859         err = devlink_nl_trap_fill(msg, devlink, trap_item,
7860                                    DEVLINK_CMD_TRAP_NEW, info->snd_portid,
7861                                    info->snd_seq, 0);
7862         if (err)
7863                 goto err_trap_fill;
7864
7865         return genlmsg_reply(msg, info);
7866
7867 err_trap_fill:
7868         nlmsg_free(msg);
7869         return err;
7870 }
7871
7872 static int devlink_nl_cmd_trap_get_dumpit(struct sk_buff *msg,
7873                                           struct netlink_callback *cb)
7874 {
7875         struct devlink_trap_item *trap_item;
7876         struct devlink *devlink;
7877         int start = cb->args[0];
7878         unsigned long index;
7879         int idx = 0;
7880         int err;
7881
7882         mutex_lock(&devlink_mutex);
7883         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
7884                 if (!devlink_try_get(devlink))
7885                         continue;
7886
7887                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
7888                         goto retry;
7889
7890                 mutex_lock(&devlink->lock);
7891                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
7892                         if (idx < start) {
7893                                 idx++;
7894                                 continue;
7895                         }
7896                         err = devlink_nl_trap_fill(msg, devlink, trap_item,
7897                                                    DEVLINK_CMD_TRAP_NEW,
7898                                                    NETLINK_CB(cb->skb).portid,
7899                                                    cb->nlh->nlmsg_seq,
7900                                                    NLM_F_MULTI);
7901                         if (err) {
7902                                 mutex_unlock(&devlink->lock);
7903                                 devlink_put(devlink);
7904                                 goto out;
7905                         }
7906                         idx++;
7907                 }
7908                 mutex_unlock(&devlink->lock);
7909 retry:
7910                 devlink_put(devlink);
7911         }
7912 out:
7913         mutex_unlock(&devlink_mutex);
7914
7915         cb->args[0] = idx;
7916         return msg->len;
7917 }
7918
7919 static int __devlink_trap_action_set(struct devlink *devlink,
7920                                      struct devlink_trap_item *trap_item,
7921                                      enum devlink_trap_action trap_action,
7922                                      struct netlink_ext_ack *extack)
7923 {
7924         int err;
7925
7926         if (trap_item->action != trap_action &&
7927             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP) {
7928                 NL_SET_ERR_MSG_MOD(extack, "Cannot change action of non-drop traps. Skipping");
7929                 return 0;
7930         }
7931
7932         err = devlink->ops->trap_action_set(devlink, trap_item->trap,
7933                                             trap_action, extack);
7934         if (err)
7935                 return err;
7936
7937         trap_item->action = trap_action;
7938
7939         return 0;
7940 }
7941
7942 static int devlink_trap_action_set(struct devlink *devlink,
7943                                    struct devlink_trap_item *trap_item,
7944                                    struct genl_info *info)
7945 {
7946         enum devlink_trap_action trap_action;
7947         int err;
7948
7949         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
7950                 return 0;
7951
7952         err = devlink_trap_action_get_from_info(info, &trap_action);
7953         if (err) {
7954                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
7955                 return -EINVAL;
7956         }
7957
7958         return __devlink_trap_action_set(devlink, trap_item, trap_action,
7959                                          info->extack);
7960 }
7961
7962 static int devlink_nl_cmd_trap_set_doit(struct sk_buff *skb,
7963                                         struct genl_info *info)
7964 {
7965         struct netlink_ext_ack *extack = info->extack;
7966         struct devlink *devlink = info->user_ptr[0];
7967         struct devlink_trap_item *trap_item;
7968
7969         if (list_empty(&devlink->trap_list))
7970                 return -EOPNOTSUPP;
7971
7972         trap_item = devlink_trap_item_get_from_info(devlink, info);
7973         if (!trap_item) {
7974                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap");
7975                 return -ENOENT;
7976         }
7977
7978         return devlink_trap_action_set(devlink, trap_item, info);
7979 }
7980
7981 static struct devlink_trap_group_item *
7982 devlink_trap_group_item_lookup(struct devlink *devlink, const char *name)
7983 {
7984         struct devlink_trap_group_item *group_item;
7985
7986         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
7987                 if (!strcmp(group_item->group->name, name))
7988                         return group_item;
7989         }
7990
7991         return NULL;
7992 }
7993
7994 static struct devlink_trap_group_item *
7995 devlink_trap_group_item_lookup_by_id(struct devlink *devlink, u16 id)
7996 {
7997         struct devlink_trap_group_item *group_item;
7998
7999         list_for_each_entry(group_item, &devlink->trap_group_list, list) {
8000                 if (group_item->group->id == id)
8001                         return group_item;
8002         }
8003
8004         return NULL;
8005 }
8006
8007 static struct devlink_trap_group_item *
8008 devlink_trap_group_item_get_from_info(struct devlink *devlink,
8009                                       struct genl_info *info)
8010 {
8011         char *name;
8012
8013         if (!info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME])
8014                 return NULL;
8015         name = nla_data(info->attrs[DEVLINK_ATTR_TRAP_GROUP_NAME]);
8016
8017         return devlink_trap_group_item_lookup(devlink, name);
8018 }
8019
8020 static int
8021 devlink_nl_trap_group_fill(struct sk_buff *msg, struct devlink *devlink,
8022                            const struct devlink_trap_group_item *group_item,
8023                            enum devlink_command cmd, u32 portid, u32 seq,
8024                            int flags)
8025 {
8026         void *hdr;
8027         int err;
8028
8029         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8030         if (!hdr)
8031                 return -EMSGSIZE;
8032
8033         if (devlink_nl_put_handle(msg, devlink))
8034                 goto nla_put_failure;
8035
8036         if (nla_put_string(msg, DEVLINK_ATTR_TRAP_GROUP_NAME,
8037                            group_item->group->name))
8038                 goto nla_put_failure;
8039
8040         if (group_item->group->generic &&
8041             nla_put_flag(msg, DEVLINK_ATTR_TRAP_GENERIC))
8042                 goto nla_put_failure;
8043
8044         if (group_item->policer_item &&
8045             nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8046                         group_item->policer_item->policer->id))
8047                 goto nla_put_failure;
8048
8049         err = devlink_trap_group_stats_put(msg, group_item->stats);
8050         if (err)
8051                 goto nla_put_failure;
8052
8053         genlmsg_end(msg, hdr);
8054
8055         return 0;
8056
8057 nla_put_failure:
8058         genlmsg_cancel(msg, hdr);
8059         return -EMSGSIZE;
8060 }
8061
8062 static int devlink_nl_cmd_trap_group_get_doit(struct sk_buff *skb,
8063                                               struct genl_info *info)
8064 {
8065         struct netlink_ext_ack *extack = info->extack;
8066         struct devlink *devlink = info->user_ptr[0];
8067         struct devlink_trap_group_item *group_item;
8068         struct sk_buff *msg;
8069         int err;
8070
8071         if (list_empty(&devlink->trap_group_list))
8072                 return -EOPNOTSUPP;
8073
8074         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8075         if (!group_item) {
8076                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8077                 return -ENOENT;
8078         }
8079
8080         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8081         if (!msg)
8082                 return -ENOMEM;
8083
8084         err = devlink_nl_trap_group_fill(msg, devlink, group_item,
8085                                          DEVLINK_CMD_TRAP_GROUP_NEW,
8086                                          info->snd_portid, info->snd_seq, 0);
8087         if (err)
8088                 goto err_trap_group_fill;
8089
8090         return genlmsg_reply(msg, info);
8091
8092 err_trap_group_fill:
8093         nlmsg_free(msg);
8094         return err;
8095 }
8096
8097 static int devlink_nl_cmd_trap_group_get_dumpit(struct sk_buff *msg,
8098                                                 struct netlink_callback *cb)
8099 {
8100         enum devlink_command cmd = DEVLINK_CMD_TRAP_GROUP_NEW;
8101         struct devlink_trap_group_item *group_item;
8102         u32 portid = NETLINK_CB(cb->skb).portid;
8103         struct devlink *devlink;
8104         int start = cb->args[0];
8105         unsigned long index;
8106         int idx = 0;
8107         int err;
8108
8109         mutex_lock(&devlink_mutex);
8110         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8111                 if (!devlink_try_get(devlink))
8112                         continue;
8113
8114                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8115                         goto retry;
8116
8117                 mutex_lock(&devlink->lock);
8118                 list_for_each_entry(group_item, &devlink->trap_group_list,
8119                                     list) {
8120                         if (idx < start) {
8121                                 idx++;
8122                                 continue;
8123                         }
8124                         err = devlink_nl_trap_group_fill(msg, devlink,
8125                                                          group_item, cmd,
8126                                                          portid,
8127                                                          cb->nlh->nlmsg_seq,
8128                                                          NLM_F_MULTI);
8129                         if (err) {
8130                                 mutex_unlock(&devlink->lock);
8131                                 devlink_put(devlink);
8132                                 goto out;
8133                         }
8134                         idx++;
8135                 }
8136                 mutex_unlock(&devlink->lock);
8137 retry:
8138                 devlink_put(devlink);
8139         }
8140 out:
8141         mutex_unlock(&devlink_mutex);
8142
8143         cb->args[0] = idx;
8144         return msg->len;
8145 }
8146
8147 static int
8148 __devlink_trap_group_action_set(struct devlink *devlink,
8149                                 struct devlink_trap_group_item *group_item,
8150                                 enum devlink_trap_action trap_action,
8151                                 struct netlink_ext_ack *extack)
8152 {
8153         const char *group_name = group_item->group->name;
8154         struct devlink_trap_item *trap_item;
8155         int err;
8156
8157         if (devlink->ops->trap_group_action_set) {
8158                 err = devlink->ops->trap_group_action_set(devlink, group_item->group,
8159                                                           trap_action, extack);
8160                 if (err)
8161                         return err;
8162
8163                 list_for_each_entry(trap_item, &devlink->trap_list, list) {
8164                         if (strcmp(trap_item->group_item->group->name, group_name))
8165                                 continue;
8166                         if (trap_item->action != trap_action &&
8167                             trap_item->trap->type != DEVLINK_TRAP_TYPE_DROP)
8168                                 continue;
8169                         trap_item->action = trap_action;
8170                 }
8171
8172                 return 0;
8173         }
8174
8175         list_for_each_entry(trap_item, &devlink->trap_list, list) {
8176                 if (strcmp(trap_item->group_item->group->name, group_name))
8177                         continue;
8178                 err = __devlink_trap_action_set(devlink, trap_item,
8179                                                 trap_action, extack);
8180                 if (err)
8181                         return err;
8182         }
8183
8184         return 0;
8185 }
8186
8187 static int
8188 devlink_trap_group_action_set(struct devlink *devlink,
8189                               struct devlink_trap_group_item *group_item,
8190                               struct genl_info *info, bool *p_modified)
8191 {
8192         enum devlink_trap_action trap_action;
8193         int err;
8194
8195         if (!info->attrs[DEVLINK_ATTR_TRAP_ACTION])
8196                 return 0;
8197
8198         err = devlink_trap_action_get_from_info(info, &trap_action);
8199         if (err) {
8200                 NL_SET_ERR_MSG_MOD(info->extack, "Invalid trap action");
8201                 return -EINVAL;
8202         }
8203
8204         err = __devlink_trap_group_action_set(devlink, group_item, trap_action,
8205                                               info->extack);
8206         if (err)
8207                 return err;
8208
8209         *p_modified = true;
8210
8211         return 0;
8212 }
8213
8214 static int devlink_trap_group_set(struct devlink *devlink,
8215                                   struct devlink_trap_group_item *group_item,
8216                                   struct genl_info *info)
8217 {
8218         struct devlink_trap_policer_item *policer_item;
8219         struct netlink_ext_ack *extack = info->extack;
8220         const struct devlink_trap_policer *policer;
8221         struct nlattr **attrs = info->attrs;
8222         int err;
8223
8224         if (!attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8225                 return 0;
8226
8227         if (!devlink->ops->trap_group_set)
8228                 return -EOPNOTSUPP;
8229
8230         policer_item = group_item->policer_item;
8231         if (attrs[DEVLINK_ATTR_TRAP_POLICER_ID]) {
8232                 u32 policer_id;
8233
8234                 policer_id = nla_get_u32(attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8235                 policer_item = devlink_trap_policer_item_lookup(devlink,
8236                                                                 policer_id);
8237                 if (policer_id && !policer_item) {
8238                         NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8239                         return -ENOENT;
8240                 }
8241         }
8242         policer = policer_item ? policer_item->policer : NULL;
8243
8244         err = devlink->ops->trap_group_set(devlink, group_item->group, policer,
8245                                            extack);
8246         if (err)
8247                 return err;
8248
8249         group_item->policer_item = policer_item;
8250
8251         return 0;
8252 }
8253
8254 static int devlink_nl_cmd_trap_group_set_doit(struct sk_buff *skb,
8255                                               struct genl_info *info)
8256 {
8257         struct netlink_ext_ack *extack = info->extack;
8258         struct devlink *devlink = info->user_ptr[0];
8259         struct devlink_trap_group_item *group_item;
8260         bool modified = false;
8261         int err;
8262
8263         if (list_empty(&devlink->trap_group_list))
8264                 return -EOPNOTSUPP;
8265
8266         group_item = devlink_trap_group_item_get_from_info(devlink, info);
8267         if (!group_item) {
8268                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap group");
8269                 return -ENOENT;
8270         }
8271
8272         err = devlink_trap_group_action_set(devlink, group_item, info,
8273                                             &modified);
8274         if (err)
8275                 return err;
8276
8277         err = devlink_trap_group_set(devlink, group_item, info);
8278         if (err)
8279                 goto err_trap_group_set;
8280
8281         return 0;
8282
8283 err_trap_group_set:
8284         if (modified)
8285                 NL_SET_ERR_MSG_MOD(extack, "Trap group set failed, but some changes were committed already");
8286         return err;
8287 }
8288
8289 static struct devlink_trap_policer_item *
8290 devlink_trap_policer_item_get_from_info(struct devlink *devlink,
8291                                         struct genl_info *info)
8292 {
8293         u32 id;
8294
8295         if (!info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID])
8296                 return NULL;
8297         id = nla_get_u32(info->attrs[DEVLINK_ATTR_TRAP_POLICER_ID]);
8298
8299         return devlink_trap_policer_item_lookup(devlink, id);
8300 }
8301
8302 static int
8303 devlink_trap_policer_stats_put(struct sk_buff *msg, struct devlink *devlink,
8304                                const struct devlink_trap_policer *policer)
8305 {
8306         struct nlattr *attr;
8307         u64 drops;
8308         int err;
8309
8310         if (!devlink->ops->trap_policer_counter_get)
8311                 return 0;
8312
8313         err = devlink->ops->trap_policer_counter_get(devlink, policer, &drops);
8314         if (err)
8315                 return err;
8316
8317         attr = nla_nest_start(msg, DEVLINK_ATTR_STATS);
8318         if (!attr)
8319                 return -EMSGSIZE;
8320
8321         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_STATS_RX_DROPPED, drops,
8322                               DEVLINK_ATTR_PAD))
8323                 goto nla_put_failure;
8324
8325         nla_nest_end(msg, attr);
8326
8327         return 0;
8328
8329 nla_put_failure:
8330         nla_nest_cancel(msg, attr);
8331         return -EMSGSIZE;
8332 }
8333
8334 static int
8335 devlink_nl_trap_policer_fill(struct sk_buff *msg, struct devlink *devlink,
8336                              const struct devlink_trap_policer_item *policer_item,
8337                              enum devlink_command cmd, u32 portid, u32 seq,
8338                              int flags)
8339 {
8340         void *hdr;
8341         int err;
8342
8343         hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
8344         if (!hdr)
8345                 return -EMSGSIZE;
8346
8347         if (devlink_nl_put_handle(msg, devlink))
8348                 goto nla_put_failure;
8349
8350         if (nla_put_u32(msg, DEVLINK_ATTR_TRAP_POLICER_ID,
8351                         policer_item->policer->id))
8352                 goto nla_put_failure;
8353
8354         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_RATE,
8355                               policer_item->rate, DEVLINK_ATTR_PAD))
8356                 goto nla_put_failure;
8357
8358         if (nla_put_u64_64bit(msg, DEVLINK_ATTR_TRAP_POLICER_BURST,
8359                               policer_item->burst, DEVLINK_ATTR_PAD))
8360                 goto nla_put_failure;
8361
8362         err = devlink_trap_policer_stats_put(msg, devlink,
8363                                              policer_item->policer);
8364         if (err)
8365                 goto nla_put_failure;
8366
8367         genlmsg_end(msg, hdr);
8368
8369         return 0;
8370
8371 nla_put_failure:
8372         genlmsg_cancel(msg, hdr);
8373         return -EMSGSIZE;
8374 }
8375
8376 static int devlink_nl_cmd_trap_policer_get_doit(struct sk_buff *skb,
8377                                                 struct genl_info *info)
8378 {
8379         struct devlink_trap_policer_item *policer_item;
8380         struct netlink_ext_ack *extack = info->extack;
8381         struct devlink *devlink = info->user_ptr[0];
8382         struct sk_buff *msg;
8383         int err;
8384
8385         if (list_empty(&devlink->trap_policer_list))
8386                 return -EOPNOTSUPP;
8387
8388         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8389         if (!policer_item) {
8390                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8391                 return -ENOENT;
8392         }
8393
8394         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
8395         if (!msg)
8396                 return -ENOMEM;
8397
8398         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item,
8399                                            DEVLINK_CMD_TRAP_POLICER_NEW,
8400                                            info->snd_portid, info->snd_seq, 0);
8401         if (err)
8402                 goto err_trap_policer_fill;
8403
8404         return genlmsg_reply(msg, info);
8405
8406 err_trap_policer_fill:
8407         nlmsg_free(msg);
8408         return err;
8409 }
8410
8411 static int devlink_nl_cmd_trap_policer_get_dumpit(struct sk_buff *msg,
8412                                                   struct netlink_callback *cb)
8413 {
8414         enum devlink_command cmd = DEVLINK_CMD_TRAP_POLICER_NEW;
8415         struct devlink_trap_policer_item *policer_item;
8416         u32 portid = NETLINK_CB(cb->skb).portid;
8417         struct devlink *devlink;
8418         int start = cb->args[0];
8419         unsigned long index;
8420         int idx = 0;
8421         int err;
8422
8423         mutex_lock(&devlink_mutex);
8424         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
8425                 if (!devlink_try_get(devlink))
8426                         continue;
8427
8428                 if (!net_eq(devlink_net(devlink), sock_net(msg->sk)))
8429                         goto retry;
8430
8431                 mutex_lock(&devlink->lock);
8432                 list_for_each_entry(policer_item, &devlink->trap_policer_list,
8433                                     list) {
8434                         if (idx < start) {
8435                                 idx++;
8436                                 continue;
8437                         }
8438                         err = devlink_nl_trap_policer_fill(msg, devlink,
8439                                                            policer_item, cmd,
8440                                                            portid,
8441                                                            cb->nlh->nlmsg_seq,
8442                                                            NLM_F_MULTI);
8443                         if (err) {
8444                                 mutex_unlock(&devlink->lock);
8445                                 devlink_put(devlink);
8446                                 goto out;
8447                         }
8448                         idx++;
8449                 }
8450                 mutex_unlock(&devlink->lock);
8451 retry:
8452                 devlink_put(devlink);
8453         }
8454 out:
8455         mutex_unlock(&devlink_mutex);
8456
8457         cb->args[0] = idx;
8458         return msg->len;
8459 }
8460
8461 static int
8462 devlink_trap_policer_set(struct devlink *devlink,
8463                          struct devlink_trap_policer_item *policer_item,
8464                          struct genl_info *info)
8465 {
8466         struct netlink_ext_ack *extack = info->extack;
8467         struct nlattr **attrs = info->attrs;
8468         u64 rate, burst;
8469         int err;
8470
8471         rate = policer_item->rate;
8472         burst = policer_item->burst;
8473
8474         if (attrs[DEVLINK_ATTR_TRAP_POLICER_RATE])
8475                 rate = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_RATE]);
8476
8477         if (attrs[DEVLINK_ATTR_TRAP_POLICER_BURST])
8478                 burst = nla_get_u64(attrs[DEVLINK_ATTR_TRAP_POLICER_BURST]);
8479
8480         if (rate < policer_item->policer->min_rate) {
8481                 NL_SET_ERR_MSG_MOD(extack, "Policer rate lower than limit");
8482                 return -EINVAL;
8483         }
8484
8485         if (rate > policer_item->policer->max_rate) {
8486                 NL_SET_ERR_MSG_MOD(extack, "Policer rate higher than limit");
8487                 return -EINVAL;
8488         }
8489
8490         if (burst < policer_item->policer->min_burst) {
8491                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size lower than limit");
8492                 return -EINVAL;
8493         }
8494
8495         if (burst > policer_item->policer->max_burst) {
8496                 NL_SET_ERR_MSG_MOD(extack, "Policer burst size higher than limit");
8497                 return -EINVAL;
8498         }
8499
8500         err = devlink->ops->trap_policer_set(devlink, policer_item->policer,
8501                                              rate, burst, info->extack);
8502         if (err)
8503                 return err;
8504
8505         policer_item->rate = rate;
8506         policer_item->burst = burst;
8507
8508         return 0;
8509 }
8510
8511 static int devlink_nl_cmd_trap_policer_set_doit(struct sk_buff *skb,
8512                                                 struct genl_info *info)
8513 {
8514         struct devlink_trap_policer_item *policer_item;
8515         struct netlink_ext_ack *extack = info->extack;
8516         struct devlink *devlink = info->user_ptr[0];
8517
8518         if (list_empty(&devlink->trap_policer_list))
8519                 return -EOPNOTSUPP;
8520
8521         if (!devlink->ops->trap_policer_set)
8522                 return -EOPNOTSUPP;
8523
8524         policer_item = devlink_trap_policer_item_get_from_info(devlink, info);
8525         if (!policer_item) {
8526                 NL_SET_ERR_MSG_MOD(extack, "Device did not register this trap policer");
8527                 return -ENOENT;
8528         }
8529
8530         return devlink_trap_policer_set(devlink, policer_item, info);
8531 }
8532
8533 static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = {
8534         [DEVLINK_ATTR_UNSPEC] = { .strict_start_type =
8535                 DEVLINK_ATTR_TRAP_POLICER_ID },
8536         [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING },
8537         [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING },
8538         [DEVLINK_ATTR_PORT_INDEX] = { .type = NLA_U32 },
8539         [DEVLINK_ATTR_PORT_TYPE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_PORT_TYPE_AUTO,
8540                                                     DEVLINK_PORT_TYPE_IB),
8541         [DEVLINK_ATTR_PORT_SPLIT_COUNT] = { .type = NLA_U32 },
8542         [DEVLINK_ATTR_SB_INDEX] = { .type = NLA_U32 },
8543         [DEVLINK_ATTR_SB_POOL_INDEX] = { .type = NLA_U16 },
8544         [DEVLINK_ATTR_SB_POOL_TYPE] = { .type = NLA_U8 },
8545         [DEVLINK_ATTR_SB_POOL_SIZE] = { .type = NLA_U32 },
8546         [DEVLINK_ATTR_SB_POOL_THRESHOLD_TYPE] = { .type = NLA_U8 },
8547         [DEVLINK_ATTR_SB_THRESHOLD] = { .type = NLA_U32 },
8548         [DEVLINK_ATTR_SB_TC_INDEX] = { .type = NLA_U16 },
8549         [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_RANGE(NLA_U16, DEVLINK_ESWITCH_MODE_LEGACY,
8550                                                        DEVLINK_ESWITCH_MODE_SWITCHDEV),
8551         [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = { .type = NLA_U8 },
8552         [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = { .type = NLA_U8 },
8553         [DEVLINK_ATTR_DPIPE_TABLE_NAME] = { .type = NLA_NUL_STRING },
8554         [DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED] = { .type = NLA_U8 },
8555         [DEVLINK_ATTR_RESOURCE_ID] = { .type = NLA_U64},
8556         [DEVLINK_ATTR_RESOURCE_SIZE] = { .type = NLA_U64},
8557         [DEVLINK_ATTR_PARAM_NAME] = { .type = NLA_NUL_STRING },
8558         [DEVLINK_ATTR_PARAM_TYPE] = { .type = NLA_U8 },
8559         [DEVLINK_ATTR_PARAM_VALUE_CMODE] = { .type = NLA_U8 },
8560         [DEVLINK_ATTR_REGION_NAME] = { .type = NLA_NUL_STRING },
8561         [DEVLINK_ATTR_REGION_SNAPSHOT_ID] = { .type = NLA_U32 },
8562         [DEVLINK_ATTR_REGION_CHUNK_ADDR] = { .type = NLA_U64 },
8563         [DEVLINK_ATTR_REGION_CHUNK_LEN] = { .type = NLA_U64 },
8564         [DEVLINK_ATTR_HEALTH_REPORTER_NAME] = { .type = NLA_NUL_STRING },
8565         [DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] = { .type = NLA_U64 },
8566         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER] = { .type = NLA_U8 },
8567         [DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME] = { .type = NLA_NUL_STRING },
8568         [DEVLINK_ATTR_FLASH_UPDATE_COMPONENT] = { .type = NLA_NUL_STRING },
8569         [DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK] =
8570                 NLA_POLICY_BITFIELD32(DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS),
8571         [DEVLINK_ATTR_TRAP_NAME] = { .type = NLA_NUL_STRING },
8572         [DEVLINK_ATTR_TRAP_ACTION] = { .type = NLA_U8 },
8573         [DEVLINK_ATTR_TRAP_GROUP_NAME] = { .type = NLA_NUL_STRING },
8574         [DEVLINK_ATTR_NETNS_PID] = { .type = NLA_U32 },
8575         [DEVLINK_ATTR_NETNS_FD] = { .type = NLA_U32 },
8576         [DEVLINK_ATTR_NETNS_ID] = { .type = NLA_U32 },
8577         [DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP] = { .type = NLA_U8 },
8578         [DEVLINK_ATTR_TRAP_POLICER_ID] = { .type = NLA_U32 },
8579         [DEVLINK_ATTR_TRAP_POLICER_RATE] = { .type = NLA_U64 },
8580         [DEVLINK_ATTR_TRAP_POLICER_BURST] = { .type = NLA_U64 },
8581         [DEVLINK_ATTR_PORT_FUNCTION] = { .type = NLA_NESTED },
8582         [DEVLINK_ATTR_RELOAD_ACTION] = NLA_POLICY_RANGE(NLA_U8, DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
8583                                                         DEVLINK_RELOAD_ACTION_MAX),
8584         [DEVLINK_ATTR_RELOAD_LIMITS] = NLA_POLICY_BITFIELD32(DEVLINK_RELOAD_LIMITS_VALID_MASK),
8585         [DEVLINK_ATTR_PORT_FLAVOUR] = { .type = NLA_U16 },
8586         [DEVLINK_ATTR_PORT_PCI_PF_NUMBER] = { .type = NLA_U16 },
8587         [DEVLINK_ATTR_PORT_PCI_SF_NUMBER] = { .type = NLA_U32 },
8588         [DEVLINK_ATTR_PORT_CONTROLLER_NUMBER] = { .type = NLA_U32 },
8589         [DEVLINK_ATTR_RATE_TYPE] = { .type = NLA_U16 },
8590         [DEVLINK_ATTR_RATE_TX_SHARE] = { .type = NLA_U64 },
8591         [DEVLINK_ATTR_RATE_TX_MAX] = { .type = NLA_U64 },
8592         [DEVLINK_ATTR_RATE_NODE_NAME] = { .type = NLA_NUL_STRING },
8593         [DEVLINK_ATTR_RATE_PARENT_NODE_NAME] = { .type = NLA_NUL_STRING },
8594 };
8595
8596 static const struct genl_small_ops devlink_nl_ops[] = {
8597         {
8598                 .cmd = DEVLINK_CMD_GET,
8599                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8600                 .doit = devlink_nl_cmd_get_doit,
8601                 .dumpit = devlink_nl_cmd_get_dumpit,
8602                 /* can be retrieved by unprivileged users */
8603         },
8604         {
8605                 .cmd = DEVLINK_CMD_PORT_GET,
8606                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8607                 .doit = devlink_nl_cmd_port_get_doit,
8608                 .dumpit = devlink_nl_cmd_port_get_dumpit,
8609                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8610                 /* can be retrieved by unprivileged users */
8611         },
8612         {
8613                 .cmd = DEVLINK_CMD_PORT_SET,
8614                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8615                 .doit = devlink_nl_cmd_port_set_doit,
8616                 .flags = GENL_ADMIN_PERM,
8617                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8618         },
8619         {
8620                 .cmd = DEVLINK_CMD_RATE_GET,
8621                 .doit = devlink_nl_cmd_rate_get_doit,
8622                 .dumpit = devlink_nl_cmd_rate_get_dumpit,
8623                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
8624                 /* can be retrieved by unprivileged users */
8625         },
8626         {
8627                 .cmd = DEVLINK_CMD_RATE_SET,
8628                 .doit = devlink_nl_cmd_rate_set_doit,
8629                 .flags = GENL_ADMIN_PERM,
8630                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE,
8631         },
8632         {
8633                 .cmd = DEVLINK_CMD_RATE_NEW,
8634                 .doit = devlink_nl_cmd_rate_new_doit,
8635                 .flags = GENL_ADMIN_PERM,
8636         },
8637         {
8638                 .cmd = DEVLINK_CMD_RATE_DEL,
8639                 .doit = devlink_nl_cmd_rate_del_doit,
8640                 .flags = GENL_ADMIN_PERM,
8641                 .internal_flags = DEVLINK_NL_FLAG_NEED_RATE_NODE,
8642         },
8643         {
8644                 .cmd = DEVLINK_CMD_PORT_SPLIT,
8645                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8646                 .doit = devlink_nl_cmd_port_split_doit,
8647                 .flags = GENL_ADMIN_PERM,
8648                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8649         },
8650         {
8651                 .cmd = DEVLINK_CMD_PORT_UNSPLIT,
8652                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8653                 .doit = devlink_nl_cmd_port_unsplit_doit,
8654                 .flags = GENL_ADMIN_PERM,
8655                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8656         },
8657         {
8658                 .cmd = DEVLINK_CMD_PORT_NEW,
8659                 .doit = devlink_nl_cmd_port_new_doit,
8660                 .flags = GENL_ADMIN_PERM,
8661                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8662         },
8663         {
8664                 .cmd = DEVLINK_CMD_PORT_DEL,
8665                 .doit = devlink_nl_cmd_port_del_doit,
8666                 .flags = GENL_ADMIN_PERM,
8667                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8668         },
8669         {
8670                 .cmd = DEVLINK_CMD_SB_GET,
8671                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8672                 .doit = devlink_nl_cmd_sb_get_doit,
8673                 .dumpit = devlink_nl_cmd_sb_get_dumpit,
8674                 /* can be retrieved by unprivileged users */
8675         },
8676         {
8677                 .cmd = DEVLINK_CMD_SB_POOL_GET,
8678                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8679                 .doit = devlink_nl_cmd_sb_pool_get_doit,
8680                 .dumpit = devlink_nl_cmd_sb_pool_get_dumpit,
8681                 /* can be retrieved by unprivileged users */
8682         },
8683         {
8684                 .cmd = DEVLINK_CMD_SB_POOL_SET,
8685                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8686                 .doit = devlink_nl_cmd_sb_pool_set_doit,
8687                 .flags = GENL_ADMIN_PERM,
8688         },
8689         {
8690                 .cmd = DEVLINK_CMD_SB_PORT_POOL_GET,
8691                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8692                 .doit = devlink_nl_cmd_sb_port_pool_get_doit,
8693                 .dumpit = devlink_nl_cmd_sb_port_pool_get_dumpit,
8694                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8695                 /* can be retrieved by unprivileged users */
8696         },
8697         {
8698                 .cmd = DEVLINK_CMD_SB_PORT_POOL_SET,
8699                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8700                 .doit = devlink_nl_cmd_sb_port_pool_set_doit,
8701                 .flags = GENL_ADMIN_PERM,
8702                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8703         },
8704         {
8705                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_GET,
8706                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8707                 .doit = devlink_nl_cmd_sb_tc_pool_bind_get_doit,
8708                 .dumpit = devlink_nl_cmd_sb_tc_pool_bind_get_dumpit,
8709                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8710                 /* can be retrieved by unprivileged users */
8711         },
8712         {
8713                 .cmd = DEVLINK_CMD_SB_TC_POOL_BIND_SET,
8714                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8715                 .doit = devlink_nl_cmd_sb_tc_pool_bind_set_doit,
8716                 .flags = GENL_ADMIN_PERM,
8717                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8718         },
8719         {
8720                 .cmd = DEVLINK_CMD_SB_OCC_SNAPSHOT,
8721                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8722                 .doit = devlink_nl_cmd_sb_occ_snapshot_doit,
8723                 .flags = GENL_ADMIN_PERM,
8724         },
8725         {
8726                 .cmd = DEVLINK_CMD_SB_OCC_MAX_CLEAR,
8727                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8728                 .doit = devlink_nl_cmd_sb_occ_max_clear_doit,
8729                 .flags = GENL_ADMIN_PERM,
8730         },
8731         {
8732                 .cmd = DEVLINK_CMD_ESWITCH_GET,
8733                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8734                 .doit = devlink_nl_cmd_eswitch_get_doit,
8735                 .flags = GENL_ADMIN_PERM,
8736                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8737         },
8738         {
8739                 .cmd = DEVLINK_CMD_ESWITCH_SET,
8740                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8741                 .doit = devlink_nl_cmd_eswitch_set_doit,
8742                 .flags = GENL_ADMIN_PERM,
8743                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8744         },
8745         {
8746                 .cmd = DEVLINK_CMD_DPIPE_TABLE_GET,
8747                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8748                 .doit = devlink_nl_cmd_dpipe_table_get,
8749                 /* can be retrieved by unprivileged users */
8750         },
8751         {
8752                 .cmd = DEVLINK_CMD_DPIPE_ENTRIES_GET,
8753                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8754                 .doit = devlink_nl_cmd_dpipe_entries_get,
8755                 /* can be retrieved by unprivileged users */
8756         },
8757         {
8758                 .cmd = DEVLINK_CMD_DPIPE_HEADERS_GET,
8759                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8760                 .doit = devlink_nl_cmd_dpipe_headers_get,
8761                 /* can be retrieved by unprivileged users */
8762         },
8763         {
8764                 .cmd = DEVLINK_CMD_DPIPE_TABLE_COUNTERS_SET,
8765                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8766                 .doit = devlink_nl_cmd_dpipe_table_counters_set,
8767                 .flags = GENL_ADMIN_PERM,
8768         },
8769         {
8770                 .cmd = DEVLINK_CMD_RESOURCE_SET,
8771                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8772                 .doit = devlink_nl_cmd_resource_set,
8773                 .flags = GENL_ADMIN_PERM,
8774         },
8775         {
8776                 .cmd = DEVLINK_CMD_RESOURCE_DUMP,
8777                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8778                 .doit = devlink_nl_cmd_resource_dump,
8779                 /* can be retrieved by unprivileged users */
8780         },
8781         {
8782                 .cmd = DEVLINK_CMD_RELOAD,
8783                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8784                 .doit = devlink_nl_cmd_reload,
8785                 .flags = GENL_ADMIN_PERM,
8786                 .internal_flags = DEVLINK_NL_FLAG_NO_LOCK,
8787         },
8788         {
8789                 .cmd = DEVLINK_CMD_PARAM_GET,
8790                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8791                 .doit = devlink_nl_cmd_param_get_doit,
8792                 .dumpit = devlink_nl_cmd_param_get_dumpit,
8793                 /* can be retrieved by unprivileged users */
8794         },
8795         {
8796                 .cmd = DEVLINK_CMD_PARAM_SET,
8797                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8798                 .doit = devlink_nl_cmd_param_set_doit,
8799                 .flags = GENL_ADMIN_PERM,
8800         },
8801         {
8802                 .cmd = DEVLINK_CMD_PORT_PARAM_GET,
8803                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8804                 .doit = devlink_nl_cmd_port_param_get_doit,
8805                 .dumpit = devlink_nl_cmd_port_param_get_dumpit,
8806                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8807                 /* can be retrieved by unprivileged users */
8808         },
8809         {
8810                 .cmd = DEVLINK_CMD_PORT_PARAM_SET,
8811                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8812                 .doit = devlink_nl_cmd_port_param_set_doit,
8813                 .flags = GENL_ADMIN_PERM,
8814                 .internal_flags = DEVLINK_NL_FLAG_NEED_PORT,
8815         },
8816         {
8817                 .cmd = DEVLINK_CMD_REGION_GET,
8818                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8819                 .doit = devlink_nl_cmd_region_get_doit,
8820                 .dumpit = devlink_nl_cmd_region_get_dumpit,
8821                 .flags = GENL_ADMIN_PERM,
8822         },
8823         {
8824                 .cmd = DEVLINK_CMD_REGION_NEW,
8825                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8826                 .doit = devlink_nl_cmd_region_new,
8827                 .flags = GENL_ADMIN_PERM,
8828         },
8829         {
8830                 .cmd = DEVLINK_CMD_REGION_DEL,
8831                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8832                 .doit = devlink_nl_cmd_region_del,
8833                 .flags = GENL_ADMIN_PERM,
8834         },
8835         {
8836                 .cmd = DEVLINK_CMD_REGION_READ,
8837                 .validate = GENL_DONT_VALIDATE_STRICT |
8838                             GENL_DONT_VALIDATE_DUMP_STRICT,
8839                 .dumpit = devlink_nl_cmd_region_read_dumpit,
8840                 .flags = GENL_ADMIN_PERM,
8841         },
8842         {
8843                 .cmd = DEVLINK_CMD_INFO_GET,
8844                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8845                 .doit = devlink_nl_cmd_info_get_doit,
8846                 .dumpit = devlink_nl_cmd_info_get_dumpit,
8847                 /* can be retrieved by unprivileged users */
8848         },
8849         {
8850                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_GET,
8851                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8852                 .doit = devlink_nl_cmd_health_reporter_get_doit,
8853                 .dumpit = devlink_nl_cmd_health_reporter_get_dumpit,
8854                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8855                                   DEVLINK_NL_FLAG_NO_LOCK,
8856                 /* can be retrieved by unprivileged users */
8857         },
8858         {
8859                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_SET,
8860                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8861                 .doit = devlink_nl_cmd_health_reporter_set_doit,
8862                 .flags = GENL_ADMIN_PERM,
8863                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8864                                   DEVLINK_NL_FLAG_NO_LOCK,
8865         },
8866         {
8867                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_RECOVER,
8868                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8869                 .doit = devlink_nl_cmd_health_reporter_recover_doit,
8870                 .flags = GENL_ADMIN_PERM,
8871                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8872                                   DEVLINK_NL_FLAG_NO_LOCK,
8873         },
8874         {
8875                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE,
8876                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8877                 .doit = devlink_nl_cmd_health_reporter_diagnose_doit,
8878                 .flags = GENL_ADMIN_PERM,
8879                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8880                                   DEVLINK_NL_FLAG_NO_LOCK,
8881         },
8882         {
8883                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET,
8884                 .validate = GENL_DONT_VALIDATE_STRICT |
8885                             GENL_DONT_VALIDATE_DUMP_STRICT,
8886                 .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit,
8887                 .flags = GENL_ADMIN_PERM,
8888         },
8889         {
8890                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR,
8891                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8892                 .doit = devlink_nl_cmd_health_reporter_dump_clear_doit,
8893                 .flags = GENL_ADMIN_PERM,
8894                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8895                                   DEVLINK_NL_FLAG_NO_LOCK,
8896         },
8897         {
8898                 .cmd = DEVLINK_CMD_HEALTH_REPORTER_TEST,
8899                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8900                 .doit = devlink_nl_cmd_health_reporter_test_doit,
8901                 .flags = GENL_ADMIN_PERM,
8902                 .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT |
8903                                   DEVLINK_NL_FLAG_NO_LOCK,
8904         },
8905         {
8906                 .cmd = DEVLINK_CMD_FLASH_UPDATE,
8907                 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
8908                 .doit = devlink_nl_cmd_flash_update,
8909                 .flags = GENL_ADMIN_PERM,
8910         },
8911         {
8912                 .cmd = DEVLINK_CMD_TRAP_GET,
8913                 .doit = devlink_nl_cmd_trap_get_doit,
8914                 .dumpit = devlink_nl_cmd_trap_get_dumpit,
8915                 /* can be retrieved by unprivileged users */
8916         },
8917         {
8918                 .cmd = DEVLINK_CMD_TRAP_SET,
8919                 .doit = devlink_nl_cmd_trap_set_doit,
8920                 .flags = GENL_ADMIN_PERM,
8921         },
8922         {
8923                 .cmd = DEVLINK_CMD_TRAP_GROUP_GET,
8924                 .doit = devlink_nl_cmd_trap_group_get_doit,
8925                 .dumpit = devlink_nl_cmd_trap_group_get_dumpit,
8926                 /* can be retrieved by unprivileged users */
8927         },
8928         {
8929                 .cmd = DEVLINK_CMD_TRAP_GROUP_SET,
8930                 .doit = devlink_nl_cmd_trap_group_set_doit,
8931                 .flags = GENL_ADMIN_PERM,
8932         },
8933         {
8934                 .cmd = DEVLINK_CMD_TRAP_POLICER_GET,
8935                 .doit = devlink_nl_cmd_trap_policer_get_doit,
8936                 .dumpit = devlink_nl_cmd_trap_policer_get_dumpit,
8937                 /* can be retrieved by unprivileged users */
8938         },
8939         {
8940                 .cmd = DEVLINK_CMD_TRAP_POLICER_SET,
8941                 .doit = devlink_nl_cmd_trap_policer_set_doit,
8942                 .flags = GENL_ADMIN_PERM,
8943         },
8944 };
8945
8946 static struct genl_family devlink_nl_family __ro_after_init = {
8947         .name           = DEVLINK_GENL_NAME,
8948         .version        = DEVLINK_GENL_VERSION,
8949         .maxattr        = DEVLINK_ATTR_MAX,
8950         .policy = devlink_nl_policy,
8951         .netnsok        = true,
8952         .pre_doit       = devlink_nl_pre_doit,
8953         .post_doit      = devlink_nl_post_doit,
8954         .module         = THIS_MODULE,
8955         .small_ops      = devlink_nl_ops,
8956         .n_small_ops    = ARRAY_SIZE(devlink_nl_ops),
8957         .mcgrps         = devlink_nl_mcgrps,
8958         .n_mcgrps       = ARRAY_SIZE(devlink_nl_mcgrps),
8959 };
8960
8961 static bool devlink_reload_actions_valid(const struct devlink_ops *ops)
8962 {
8963         const struct devlink_reload_combination *comb;
8964         int i;
8965
8966         if (!devlink_reload_supported(ops)) {
8967                 if (WARN_ON(ops->reload_actions))
8968                         return false;
8969                 return true;
8970         }
8971
8972         if (WARN_ON(!ops->reload_actions ||
8973                     ops->reload_actions & BIT(DEVLINK_RELOAD_ACTION_UNSPEC) ||
8974                     ops->reload_actions >= BIT(__DEVLINK_RELOAD_ACTION_MAX)))
8975                 return false;
8976
8977         if (WARN_ON(ops->reload_limits & BIT(DEVLINK_RELOAD_LIMIT_UNSPEC) ||
8978                     ops->reload_limits >= BIT(__DEVLINK_RELOAD_LIMIT_MAX)))
8979                 return false;
8980
8981         for (i = 0; i < ARRAY_SIZE(devlink_reload_invalid_combinations); i++)  {
8982                 comb = &devlink_reload_invalid_combinations[i];
8983                 if (ops->reload_actions == BIT(comb->action) &&
8984                     ops->reload_limits == BIT(comb->limit))
8985                         return false;
8986         }
8987         return true;
8988 }
8989
8990 /**
8991  *      devlink_set_features - Set devlink supported features
8992  *
8993  *      @devlink: devlink
8994  *      @features: devlink support features
8995  *
8996  *      This interface allows us to set reload ops separatelly from
8997  *      the devlink_alloc.
8998  */
8999 void devlink_set_features(struct devlink *devlink, u64 features)
9000 {
9001         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9002
9003         WARN_ON(features & DEVLINK_F_RELOAD &&
9004                 !devlink_reload_supported(devlink->ops));
9005         devlink->features = features;
9006 }
9007 EXPORT_SYMBOL_GPL(devlink_set_features);
9008
9009 /**
9010  *      devlink_alloc_ns - Allocate new devlink instance resources
9011  *      in specific namespace
9012  *
9013  *      @ops: ops
9014  *      @priv_size: size of user private data
9015  *      @net: net namespace
9016  *      @dev: parent device
9017  *
9018  *      Allocate new devlink instance resources, including devlink index
9019  *      and name.
9020  */
9021 struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
9022                                  size_t priv_size, struct net *net,
9023                                  struct device *dev)
9024 {
9025         struct devlink *devlink;
9026         static u32 last_id;
9027         int ret;
9028
9029         WARN_ON(!ops || !dev);
9030         if (!devlink_reload_actions_valid(ops))
9031                 return NULL;
9032
9033         devlink = kzalloc(sizeof(*devlink) + priv_size, GFP_KERNEL);
9034         if (!devlink)
9035                 return NULL;
9036
9037         ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
9038                               &last_id, GFP_KERNEL);
9039         if (ret < 0) {
9040                 kfree(devlink);
9041                 return NULL;
9042         }
9043
9044         devlink->dev = dev;
9045         devlink->ops = ops;
9046         xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
9047         write_pnet(&devlink->_net, net);
9048         INIT_LIST_HEAD(&devlink->port_list);
9049         INIT_LIST_HEAD(&devlink->rate_list);
9050         INIT_LIST_HEAD(&devlink->sb_list);
9051         INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
9052         INIT_LIST_HEAD(&devlink->resource_list);
9053         INIT_LIST_HEAD(&devlink->param_list);
9054         INIT_LIST_HEAD(&devlink->region_list);
9055         INIT_LIST_HEAD(&devlink->reporter_list);
9056         INIT_LIST_HEAD(&devlink->trap_list);
9057         INIT_LIST_HEAD(&devlink->trap_group_list);
9058         INIT_LIST_HEAD(&devlink->trap_policer_list);
9059         mutex_init(&devlink->lock);
9060         mutex_init(&devlink->reporters_lock);
9061         refcount_set(&devlink->refcount, 1);
9062         init_completion(&devlink->comp);
9063
9064         return devlink;
9065 }
9066 EXPORT_SYMBOL_GPL(devlink_alloc_ns);
9067
9068 static void
9069 devlink_trap_policer_notify(struct devlink *devlink,
9070                             const struct devlink_trap_policer_item *policer_item,
9071                             enum devlink_command cmd);
9072 static void
9073 devlink_trap_group_notify(struct devlink *devlink,
9074                           const struct devlink_trap_group_item *group_item,
9075                           enum devlink_command cmd);
9076 static void devlink_trap_notify(struct devlink *devlink,
9077                                 const struct devlink_trap_item *trap_item,
9078                                 enum devlink_command cmd);
9079
9080 static void devlink_notify_register(struct devlink *devlink)
9081 {
9082         struct devlink_trap_policer_item *policer_item;
9083         struct devlink_trap_group_item *group_item;
9084         struct devlink_param_item *param_item;
9085         struct devlink_trap_item *trap_item;
9086         struct devlink_port *devlink_port;
9087         struct devlink_rate *rate_node;
9088         struct devlink_region *region;
9089
9090         devlink_notify(devlink, DEVLINK_CMD_NEW);
9091         list_for_each_entry(devlink_port, &devlink->port_list, list)
9092                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9093
9094         list_for_each_entry(policer_item, &devlink->trap_policer_list, list)
9095                 devlink_trap_policer_notify(devlink, policer_item,
9096                                             DEVLINK_CMD_TRAP_POLICER_NEW);
9097
9098         list_for_each_entry(group_item, &devlink->trap_group_list, list)
9099                 devlink_trap_group_notify(devlink, group_item,
9100                                           DEVLINK_CMD_TRAP_GROUP_NEW);
9101
9102         list_for_each_entry(trap_item, &devlink->trap_list, list)
9103                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
9104
9105         list_for_each_entry(rate_node, &devlink->rate_list, list)
9106                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_NEW);
9107
9108         list_for_each_entry(region, &devlink->region_list, list)
9109                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
9110
9111         list_for_each_entry(param_item, &devlink->param_list, list)
9112                 devlink_param_notify(devlink, 0, param_item,
9113                                      DEVLINK_CMD_PARAM_NEW);
9114 }
9115
9116 static void devlink_notify_unregister(struct devlink *devlink)
9117 {
9118         struct devlink_trap_policer_item *policer_item;
9119         struct devlink_trap_group_item *group_item;
9120         struct devlink_param_item *param_item;
9121         struct devlink_trap_item *trap_item;
9122         struct devlink_port *devlink_port;
9123         struct devlink_rate *rate_node;
9124         struct devlink_region *region;
9125
9126         list_for_each_entry_reverse(param_item, &devlink->param_list, list)
9127                 devlink_param_notify(devlink, 0, param_item,
9128                                      DEVLINK_CMD_PARAM_DEL);
9129
9130         list_for_each_entry_reverse(region, &devlink->region_list, list)
9131                 devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
9132
9133         list_for_each_entry_reverse(rate_node, &devlink->rate_list, list)
9134                 devlink_rate_notify(rate_node, DEVLINK_CMD_RATE_DEL);
9135
9136         list_for_each_entry_reverse(trap_item, &devlink->trap_list, list)
9137                 devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
9138
9139         list_for_each_entry_reverse(group_item, &devlink->trap_group_list, list)
9140                 devlink_trap_group_notify(devlink, group_item,
9141                                           DEVLINK_CMD_TRAP_GROUP_DEL);
9142         list_for_each_entry_reverse(policer_item, &devlink->trap_policer_list,
9143                                     list)
9144                 devlink_trap_policer_notify(devlink, policer_item,
9145                                             DEVLINK_CMD_TRAP_POLICER_DEL);
9146
9147         list_for_each_entry_reverse(devlink_port, &devlink->port_list, list)
9148                 devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9149         devlink_notify(devlink, DEVLINK_CMD_DEL);
9150 }
9151
9152 /**
9153  *      devlink_register - Register devlink instance
9154  *
9155  *      @devlink: devlink
9156  */
9157 void devlink_register(struct devlink *devlink)
9158 {
9159         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9160         /* Make sure that we are in .probe() routine */
9161
9162         mutex_lock(&devlink_mutex);
9163         xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9164         devlink_notify_register(devlink);
9165         mutex_unlock(&devlink_mutex);
9166 }
9167 EXPORT_SYMBOL_GPL(devlink_register);
9168
9169 /**
9170  *      devlink_unregister - Unregister devlink instance
9171  *
9172  *      @devlink: devlink
9173  */
9174 void devlink_unregister(struct devlink *devlink)
9175 {
9176         ASSERT_DEVLINK_REGISTERED(devlink);
9177         /* Make sure that we are in .remove() routine */
9178
9179         devlink_put(devlink);
9180         wait_for_completion(&devlink->comp);
9181
9182         mutex_lock(&devlink_mutex);
9183         devlink_notify_unregister(devlink);
9184         xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
9185         mutex_unlock(&devlink_mutex);
9186 }
9187 EXPORT_SYMBOL_GPL(devlink_unregister);
9188
9189 /**
9190  *      devlink_free - Free devlink instance resources
9191  *
9192  *      @devlink: devlink
9193  */
9194 void devlink_free(struct devlink *devlink)
9195 {
9196         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
9197
9198         mutex_destroy(&devlink->reporters_lock);
9199         mutex_destroy(&devlink->lock);
9200         WARN_ON(!list_empty(&devlink->trap_policer_list));
9201         WARN_ON(!list_empty(&devlink->trap_group_list));
9202         WARN_ON(!list_empty(&devlink->trap_list));
9203         WARN_ON(!list_empty(&devlink->reporter_list));
9204         WARN_ON(!list_empty(&devlink->region_list));
9205         WARN_ON(!list_empty(&devlink->param_list));
9206         WARN_ON(!list_empty(&devlink->resource_list));
9207         WARN_ON(!list_empty(&devlink->dpipe_table_list));
9208         WARN_ON(!list_empty(&devlink->sb_list));
9209         WARN_ON(!list_empty(&devlink->rate_list));
9210         WARN_ON(!list_empty(&devlink->port_list));
9211
9212         xa_destroy(&devlink->snapshot_ids);
9213         xa_erase(&devlinks, devlink->index);
9214
9215         kfree(devlink);
9216 }
9217 EXPORT_SYMBOL_GPL(devlink_free);
9218
9219 static void devlink_port_type_warn(struct work_struct *work)
9220 {
9221         WARN(true, "Type was not set for devlink port.");
9222 }
9223
9224 static bool devlink_port_type_should_warn(struct devlink_port *devlink_port)
9225 {
9226         /* Ignore CPU and DSA flavours. */
9227         return devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_CPU &&
9228                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_DSA &&
9229                devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_UNUSED;
9230 }
9231
9232 #define DEVLINK_PORT_TYPE_WARN_TIMEOUT (HZ * 3600)
9233
9234 static void devlink_port_type_warn_schedule(struct devlink_port *devlink_port)
9235 {
9236         if (!devlink_port_type_should_warn(devlink_port))
9237                 return;
9238         /* Schedule a work to WARN in case driver does not set port
9239          * type within timeout.
9240          */
9241         schedule_delayed_work(&devlink_port->type_warn_dw,
9242                               DEVLINK_PORT_TYPE_WARN_TIMEOUT);
9243 }
9244
9245 static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
9246 {
9247         if (!devlink_port_type_should_warn(devlink_port))
9248                 return;
9249         cancel_delayed_work_sync(&devlink_port->type_warn_dw);
9250 }
9251
9252 /**
9253  *      devlink_port_register - Register devlink port
9254  *
9255  *      @devlink: devlink
9256  *      @devlink_port: devlink port
9257  *      @port_index: driver-specific numerical identifier of the port
9258  *
9259  *      Register devlink port with provided port index. User can use
9260  *      any indexing, even hw-related one. devlink_port structure
9261  *      is convenient to be embedded inside user driver private structure.
9262  *      Note that the caller should take care of zeroing the devlink_port
9263  *      structure.
9264  */
9265 int devlink_port_register(struct devlink *devlink,
9266                           struct devlink_port *devlink_port,
9267                           unsigned int port_index)
9268 {
9269         mutex_lock(&devlink->lock);
9270         if (devlink_port_index_exists(devlink, port_index)) {
9271                 mutex_unlock(&devlink->lock);
9272                 return -EEXIST;
9273         }
9274
9275         WARN_ON(devlink_port->devlink);
9276         devlink_port->devlink = devlink;
9277         devlink_port->index = port_index;
9278         spin_lock_init(&devlink_port->type_lock);
9279         INIT_LIST_HEAD(&devlink_port->reporter_list);
9280         mutex_init(&devlink_port->reporters_lock);
9281         list_add_tail(&devlink_port->list, &devlink->port_list);
9282         INIT_LIST_HEAD(&devlink_port->param_list);
9283         INIT_LIST_HEAD(&devlink_port->region_list);
9284         mutex_unlock(&devlink->lock);
9285         INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
9286         devlink_port_type_warn_schedule(devlink_port);
9287         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9288         return 0;
9289 }
9290 EXPORT_SYMBOL_GPL(devlink_port_register);
9291
9292 /**
9293  *      devlink_port_unregister - Unregister devlink port
9294  *
9295  *      @devlink_port: devlink port
9296  */
9297 void devlink_port_unregister(struct devlink_port *devlink_port)
9298 {
9299         struct devlink *devlink = devlink_port->devlink;
9300
9301         devlink_port_type_warn_cancel(devlink_port);
9302         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
9303         mutex_lock(&devlink->lock);
9304         list_del(&devlink_port->list);
9305         mutex_unlock(&devlink->lock);
9306         WARN_ON(!list_empty(&devlink_port->reporter_list));
9307         WARN_ON(!list_empty(&devlink_port->region_list));
9308         mutex_destroy(&devlink_port->reporters_lock);
9309 }
9310 EXPORT_SYMBOL_GPL(devlink_port_unregister);
9311
9312 static void __devlink_port_type_set(struct devlink_port *devlink_port,
9313                                     enum devlink_port_type type,
9314                                     void *type_dev)
9315 {
9316         if (WARN_ON(!devlink_port->devlink))
9317                 return;
9318         devlink_port_type_warn_cancel(devlink_port);
9319         spin_lock_bh(&devlink_port->type_lock);
9320         devlink_port->type = type;
9321         devlink_port->type_dev = type_dev;
9322         spin_unlock_bh(&devlink_port->type_lock);
9323         devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW);
9324 }
9325
9326 static void devlink_port_type_netdev_checks(struct devlink_port *devlink_port,
9327                                             struct net_device *netdev)
9328 {
9329         const struct net_device_ops *ops = netdev->netdev_ops;
9330
9331         /* If driver registers devlink port, it should set devlink port
9332          * attributes accordingly so the compat functions are called
9333          * and the original ops are not used.
9334          */
9335         if (ops->ndo_get_phys_port_name) {
9336                 /* Some drivers use the same set of ndos for netdevs
9337                  * that have devlink_port registered and also for
9338                  * those who don't. Make sure that ndo_get_phys_port_name
9339                  * returns -EOPNOTSUPP here in case it is defined.
9340                  * Warn if not.
9341                  */
9342                 char name[IFNAMSIZ];
9343                 int err;
9344
9345                 err = ops->ndo_get_phys_port_name(netdev, name, sizeof(name));
9346                 WARN_ON(err != -EOPNOTSUPP);
9347         }
9348         if (ops->ndo_get_port_parent_id) {
9349                 /* Some drivers use the same set of ndos for netdevs
9350                  * that have devlink_port registered and also for
9351                  * those who don't. Make sure that ndo_get_port_parent_id
9352                  * returns -EOPNOTSUPP here in case it is defined.
9353                  * Warn if not.
9354                  */
9355                 struct netdev_phys_item_id ppid;
9356                 int err;
9357
9358                 err = ops->ndo_get_port_parent_id(netdev, &ppid);
9359                 WARN_ON(err != -EOPNOTSUPP);
9360         }
9361 }
9362
9363 /**
9364  *      devlink_port_type_eth_set - Set port type to Ethernet
9365  *
9366  *      @devlink_port: devlink port
9367  *      @netdev: related netdevice
9368  */
9369 void devlink_port_type_eth_set(struct devlink_port *devlink_port,
9370                                struct net_device *netdev)
9371 {
9372         if (netdev)
9373                 devlink_port_type_netdev_checks(devlink_port, netdev);
9374         else
9375                 dev_warn(devlink_port->devlink->dev,
9376                          "devlink port type for port %d set to Ethernet without a software interface reference, device type not supported by the kernel?\n",
9377                          devlink_port->index);
9378
9379         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_ETH, netdev);
9380 }
9381 EXPORT_SYMBOL_GPL(devlink_port_type_eth_set);
9382
9383 /**
9384  *      devlink_port_type_ib_set - Set port type to InfiniBand
9385  *
9386  *      @devlink_port: devlink port
9387  *      @ibdev: related IB device
9388  */
9389 void devlink_port_type_ib_set(struct devlink_port *devlink_port,
9390                               struct ib_device *ibdev)
9391 {
9392         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_IB, ibdev);
9393 }
9394 EXPORT_SYMBOL_GPL(devlink_port_type_ib_set);
9395
9396 /**
9397  *      devlink_port_type_clear - Clear port type
9398  *
9399  *      @devlink_port: devlink port
9400  */
9401 void devlink_port_type_clear(struct devlink_port *devlink_port)
9402 {
9403         __devlink_port_type_set(devlink_port, DEVLINK_PORT_TYPE_NOTSET, NULL);
9404         devlink_port_type_warn_schedule(devlink_port);
9405 }
9406 EXPORT_SYMBOL_GPL(devlink_port_type_clear);
9407
9408 static int __devlink_port_attrs_set(struct devlink_port *devlink_port,
9409                                     enum devlink_port_flavour flavour)
9410 {
9411         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9412
9413         devlink_port->attrs_set = true;
9414         attrs->flavour = flavour;
9415         if (attrs->switch_id.id_len) {
9416                 devlink_port->switch_port = true;
9417                 if (WARN_ON(attrs->switch_id.id_len > MAX_PHYS_ITEM_ID_LEN))
9418                         attrs->switch_id.id_len = MAX_PHYS_ITEM_ID_LEN;
9419         } else {
9420                 devlink_port->switch_port = false;
9421         }
9422         return 0;
9423 }
9424
9425 /**
9426  *      devlink_port_attrs_set - Set port attributes
9427  *
9428  *      @devlink_port: devlink port
9429  *      @attrs: devlink port attrs
9430  */
9431 void devlink_port_attrs_set(struct devlink_port *devlink_port,
9432                             struct devlink_port_attrs *attrs)
9433 {
9434         int ret;
9435
9436         if (WARN_ON(devlink_port->devlink))
9437                 return;
9438         devlink_port->attrs = *attrs;
9439         ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
9440         if (ret)
9441                 return;
9442         WARN_ON(attrs->splittable && attrs->split);
9443 }
9444 EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
9445
9446 /**
9447  *      devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
9448  *
9449  *      @devlink_port: devlink port
9450  *      @controller: associated controller number for the devlink port instance
9451  *      @pf: associated PF for the devlink port instance
9452  *      @external: indicates if the port is for an external controller
9453  */
9454 void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
9455                                    u16 pf, bool external)
9456 {
9457         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9458         int ret;
9459
9460         if (WARN_ON(devlink_port->devlink))
9461                 return;
9462         ret = __devlink_port_attrs_set(devlink_port,
9463                                        DEVLINK_PORT_FLAVOUR_PCI_PF);
9464         if (ret)
9465                 return;
9466         attrs->pci_pf.controller = controller;
9467         attrs->pci_pf.pf = pf;
9468         attrs->pci_pf.external = external;
9469 }
9470 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
9471
9472 /**
9473  *      devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
9474  *
9475  *      @devlink_port: devlink port
9476  *      @controller: associated controller number for the devlink port instance
9477  *      @pf: associated PF for the devlink port instance
9478  *      @vf: associated VF of a PF for the devlink port instance
9479  *      @external: indicates if the port is for an external controller
9480  */
9481 void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
9482                                    u16 pf, u16 vf, bool external)
9483 {
9484         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9485         int ret;
9486
9487         if (WARN_ON(devlink_port->devlink))
9488                 return;
9489         ret = __devlink_port_attrs_set(devlink_port,
9490                                        DEVLINK_PORT_FLAVOUR_PCI_VF);
9491         if (ret)
9492                 return;
9493         attrs->pci_vf.controller = controller;
9494         attrs->pci_vf.pf = pf;
9495         attrs->pci_vf.vf = vf;
9496         attrs->pci_vf.external = external;
9497 }
9498 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_vf_set);
9499
9500 /**
9501  *      devlink_port_attrs_pci_sf_set - Set PCI SF port attributes
9502  *
9503  *      @devlink_port: devlink port
9504  *      @controller: associated controller number for the devlink port instance
9505  *      @pf: associated PF for the devlink port instance
9506  *      @sf: associated SF of a PF for the devlink port instance
9507  *      @external: indicates if the port is for an external controller
9508  */
9509 void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller,
9510                                    u16 pf, u32 sf, bool external)
9511 {
9512         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9513         int ret;
9514
9515         if (WARN_ON(devlink_port->devlink))
9516                 return;
9517         ret = __devlink_port_attrs_set(devlink_port,
9518                                        DEVLINK_PORT_FLAVOUR_PCI_SF);
9519         if (ret)
9520                 return;
9521         attrs->pci_sf.controller = controller;
9522         attrs->pci_sf.pf = pf;
9523         attrs->pci_sf.sf = sf;
9524         attrs->pci_sf.external = external;
9525 }
9526 EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set);
9527
9528 /**
9529  * devlink_rate_leaf_create - create devlink rate leaf
9530  *
9531  * @devlink_port: devlink port object to create rate object on
9532  * @priv: driver private data
9533  *
9534  * Create devlink rate object of type leaf on provided @devlink_port.
9535  * Throws call trace if @devlink_port already has a devlink rate object.
9536  *
9537  * Context: Takes and release devlink->lock <mutex>.
9538  *
9539  * Return: -ENOMEM if failed to allocate rate object, 0 otherwise.
9540  */
9541 int
9542 devlink_rate_leaf_create(struct devlink_port *devlink_port, void *priv)
9543 {
9544         struct devlink *devlink = devlink_port->devlink;
9545         struct devlink_rate *devlink_rate;
9546
9547         devlink_rate = kzalloc(sizeof(*devlink_rate), GFP_KERNEL);
9548         if (!devlink_rate)
9549                 return -ENOMEM;
9550
9551         mutex_lock(&devlink->lock);
9552         WARN_ON(devlink_port->devlink_rate);
9553         devlink_rate->type = DEVLINK_RATE_TYPE_LEAF;
9554         devlink_rate->devlink = devlink;
9555         devlink_rate->devlink_port = devlink_port;
9556         devlink_rate->priv = priv;
9557         list_add_tail(&devlink_rate->list, &devlink->rate_list);
9558         devlink_port->devlink_rate = devlink_rate;
9559         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_NEW);
9560         mutex_unlock(&devlink->lock);
9561
9562         return 0;
9563 }
9564 EXPORT_SYMBOL_GPL(devlink_rate_leaf_create);
9565
9566 /**
9567  * devlink_rate_leaf_destroy - destroy devlink rate leaf
9568  *
9569  * @devlink_port: devlink port linked to the rate object
9570  *
9571  * Context: Takes and release devlink->lock <mutex>.
9572  */
9573 void devlink_rate_leaf_destroy(struct devlink_port *devlink_port)
9574 {
9575         struct devlink_rate *devlink_rate = devlink_port->devlink_rate;
9576         struct devlink *devlink = devlink_port->devlink;
9577
9578         if (!devlink_rate)
9579                 return;
9580
9581         mutex_lock(&devlink->lock);
9582         devlink_rate_notify(devlink_rate, DEVLINK_CMD_RATE_DEL);
9583         if (devlink_rate->parent)
9584                 refcount_dec(&devlink_rate->parent->refcnt);
9585         list_del(&devlink_rate->list);
9586         devlink_port->devlink_rate = NULL;
9587         mutex_unlock(&devlink->lock);
9588         kfree(devlink_rate);
9589 }
9590 EXPORT_SYMBOL_GPL(devlink_rate_leaf_destroy);
9591
9592 /**
9593  * devlink_rate_nodes_destroy - destroy all devlink rate nodes on device
9594  *
9595  * @devlink: devlink instance
9596  *
9597  * Unset parent for all rate objects and destroy all rate nodes
9598  * on specified device.
9599  *
9600  * Context: Takes and release devlink->lock <mutex>.
9601  */
9602 void devlink_rate_nodes_destroy(struct devlink *devlink)
9603 {
9604         static struct devlink_rate *devlink_rate, *tmp;
9605         const struct devlink_ops *ops = devlink->ops;
9606
9607         mutex_lock(&devlink->lock);
9608         list_for_each_entry(devlink_rate, &devlink->rate_list, list) {
9609                 if (!devlink_rate->parent)
9610                         continue;
9611
9612                 refcount_dec(&devlink_rate->parent->refcnt);
9613                 if (devlink_rate_is_leaf(devlink_rate))
9614                         ops->rate_leaf_parent_set(devlink_rate, NULL, devlink_rate->priv,
9615                                                   NULL, NULL);
9616                 else if (devlink_rate_is_node(devlink_rate))
9617                         ops->rate_node_parent_set(devlink_rate, NULL, devlink_rate->priv,
9618                                                   NULL, NULL);
9619         }
9620         list_for_each_entry_safe(devlink_rate, tmp, &devlink->rate_list, list) {
9621                 if (devlink_rate_is_node(devlink_rate)) {
9622                         ops->rate_node_del(devlink_rate, devlink_rate->priv, NULL);
9623                         list_del(&devlink_rate->list);
9624                         kfree(devlink_rate->name);
9625                         kfree(devlink_rate);
9626                 }
9627         }
9628         mutex_unlock(&devlink->lock);
9629 }
9630 EXPORT_SYMBOL_GPL(devlink_rate_nodes_destroy);
9631
9632 static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port,
9633                                              char *name, size_t len)
9634 {
9635         struct devlink_port_attrs *attrs = &devlink_port->attrs;
9636         int n = 0;
9637
9638         if (!devlink_port->attrs_set)
9639                 return -EOPNOTSUPP;
9640
9641         switch (attrs->flavour) {
9642         case DEVLINK_PORT_FLAVOUR_PHYSICAL:
9643                 n = snprintf(name, len, "p%u", attrs->phys.port_number);
9644                 if (n < len && attrs->split)
9645                         n += snprintf(name + n, len - n, "s%u",
9646                                       attrs->phys.split_subport_number);
9647                 break;
9648         case DEVLINK_PORT_FLAVOUR_CPU:
9649         case DEVLINK_PORT_FLAVOUR_DSA:
9650         case DEVLINK_PORT_FLAVOUR_UNUSED:
9651                 /* As CPU and DSA ports do not have a netdevice associated
9652                  * case should not ever happen.
9653                  */
9654                 WARN_ON(1);
9655                 return -EINVAL;
9656         case DEVLINK_PORT_FLAVOUR_PCI_PF:
9657                 if (attrs->pci_pf.external) {
9658                         n = snprintf(name, len, "c%u", attrs->pci_pf.controller);
9659                         if (n >= len)
9660                                 return -EINVAL;
9661                         len -= n;
9662                         name += n;
9663                 }
9664                 n = snprintf(name, len, "pf%u", attrs->pci_pf.pf);
9665                 break;
9666         case DEVLINK_PORT_FLAVOUR_PCI_VF:
9667                 if (attrs->pci_vf.external) {
9668                         n = snprintf(name, len, "c%u", attrs->pci_vf.controller);
9669                         if (n >= len)
9670                                 return -EINVAL;
9671                         len -= n;
9672                         name += n;
9673                 }
9674                 n = snprintf(name, len, "pf%uvf%u",
9675                              attrs->pci_vf.pf, attrs->pci_vf.vf);
9676                 break;
9677         case DEVLINK_PORT_FLAVOUR_PCI_SF:
9678                 if (attrs->pci_sf.external) {
9679                         n = snprintf(name, len, "c%u", attrs->pci_sf.controller);
9680                         if (n >= len)
9681                                 return -EINVAL;
9682                         len -= n;
9683                         name += n;
9684                 }
9685                 n = snprintf(name, len, "pf%usf%u", attrs->pci_sf.pf,
9686                              attrs->pci_sf.sf);
9687                 break;
9688         case DEVLINK_PORT_FLAVOUR_VIRTUAL:
9689                 return -EOPNOTSUPP;
9690         }
9691
9692         if (n >= len)
9693                 return -EINVAL;
9694
9695         return 0;
9696 }
9697
9698 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
9699                         u32 size, u16 ingress_pools_count,
9700                         u16 egress_pools_count, u16 ingress_tc_count,
9701                         u16 egress_tc_count)
9702 {
9703         struct devlink_sb *devlink_sb;
9704         int err = 0;
9705
9706         mutex_lock(&devlink->lock);
9707         if (devlink_sb_index_exists(devlink, sb_index)) {
9708                 err = -EEXIST;
9709                 goto unlock;
9710         }
9711
9712         devlink_sb = kzalloc(sizeof(*devlink_sb), GFP_KERNEL);
9713         if (!devlink_sb) {
9714                 err = -ENOMEM;
9715                 goto unlock;
9716         }
9717         devlink_sb->index = sb_index;
9718         devlink_sb->size = size;
9719         devlink_sb->ingress_pools_count = ingress_pools_count;
9720         devlink_sb->egress_pools_count = egress_pools_count;
9721         devlink_sb->ingress_tc_count = ingress_tc_count;
9722         devlink_sb->egress_tc_count = egress_tc_count;
9723         list_add_tail(&devlink_sb->list, &devlink->sb_list);
9724 unlock:
9725         mutex_unlock(&devlink->lock);
9726         return err;
9727 }
9728 EXPORT_SYMBOL_GPL(devlink_sb_register);
9729
9730 void devlink_sb_unregister(struct devlink *devlink, unsigned int sb_index)
9731 {
9732         struct devlink_sb *devlink_sb;
9733
9734         mutex_lock(&devlink->lock);
9735         devlink_sb = devlink_sb_get_by_index(devlink, sb_index);
9736         WARN_ON(!devlink_sb);
9737         list_del(&devlink_sb->list);
9738         mutex_unlock(&devlink->lock);
9739         kfree(devlink_sb);
9740 }
9741 EXPORT_SYMBOL_GPL(devlink_sb_unregister);
9742
9743 /**
9744  *      devlink_dpipe_headers_register - register dpipe headers
9745  *
9746  *      @devlink: devlink
9747  *      @dpipe_headers: dpipe header array
9748  *
9749  *      Register the headers supported by hardware.
9750  */
9751 int devlink_dpipe_headers_register(struct devlink *devlink,
9752                                    struct devlink_dpipe_headers *dpipe_headers)
9753 {
9754         mutex_lock(&devlink->lock);
9755         devlink->dpipe_headers = dpipe_headers;
9756         mutex_unlock(&devlink->lock);
9757         return 0;
9758 }
9759 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_register);
9760
9761 /**
9762  *      devlink_dpipe_headers_unregister - unregister dpipe headers
9763  *
9764  *      @devlink: devlink
9765  *
9766  *      Unregister the headers supported by hardware.
9767  */
9768 void devlink_dpipe_headers_unregister(struct devlink *devlink)
9769 {
9770         mutex_lock(&devlink->lock);
9771         devlink->dpipe_headers = NULL;
9772         mutex_unlock(&devlink->lock);
9773 }
9774 EXPORT_SYMBOL_GPL(devlink_dpipe_headers_unregister);
9775
9776 /**
9777  *      devlink_dpipe_table_counter_enabled - check if counter allocation
9778  *                                            required
9779  *      @devlink: devlink
9780  *      @table_name: tables name
9781  *
9782  *      Used by driver to check if counter allocation is required.
9783  *      After counter allocation is turned on the table entries
9784  *      are updated to include counter statistics.
9785  *
9786  *      After that point on the driver must respect the counter
9787  *      state so that each entry added to the table is added
9788  *      with a counter.
9789  */
9790 bool devlink_dpipe_table_counter_enabled(struct devlink *devlink,
9791                                          const char *table_name)
9792 {
9793         struct devlink_dpipe_table *table;
9794         bool enabled;
9795
9796         rcu_read_lock();
9797         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
9798                                          table_name, devlink);
9799         enabled = false;
9800         if (table)
9801                 enabled = table->counters_enabled;
9802         rcu_read_unlock();
9803         return enabled;
9804 }
9805 EXPORT_SYMBOL_GPL(devlink_dpipe_table_counter_enabled);
9806
9807 /**
9808  *      devlink_dpipe_table_register - register dpipe table
9809  *
9810  *      @devlink: devlink
9811  *      @table_name: table name
9812  *      @table_ops: table ops
9813  *      @priv: priv
9814  *      @counter_control_extern: external control for counters
9815  */
9816 int devlink_dpipe_table_register(struct devlink *devlink,
9817                                  const char *table_name,
9818                                  struct devlink_dpipe_table_ops *table_ops,
9819                                  void *priv, bool counter_control_extern)
9820 {
9821         struct devlink_dpipe_table *table;
9822         int err = 0;
9823
9824         if (WARN_ON(!table_ops->size_get))
9825                 return -EINVAL;
9826
9827         mutex_lock(&devlink->lock);
9828
9829         if (devlink_dpipe_table_find(&devlink->dpipe_table_list, table_name,
9830                                      devlink)) {
9831                 err = -EEXIST;
9832                 goto unlock;
9833         }
9834
9835         table = kzalloc(sizeof(*table), GFP_KERNEL);
9836         if (!table) {
9837                 err = -ENOMEM;
9838                 goto unlock;
9839         }
9840
9841         table->name = table_name;
9842         table->table_ops = table_ops;
9843         table->priv = priv;
9844         table->counter_control_extern = counter_control_extern;
9845
9846         list_add_tail_rcu(&table->list, &devlink->dpipe_table_list);
9847 unlock:
9848         mutex_unlock(&devlink->lock);
9849         return err;
9850 }
9851 EXPORT_SYMBOL_GPL(devlink_dpipe_table_register);
9852
9853 /**
9854  *      devlink_dpipe_table_unregister - unregister dpipe table
9855  *
9856  *      @devlink: devlink
9857  *      @table_name: table name
9858  */
9859 void devlink_dpipe_table_unregister(struct devlink *devlink,
9860                                     const char *table_name)
9861 {
9862         struct devlink_dpipe_table *table;
9863
9864         mutex_lock(&devlink->lock);
9865         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
9866                                          table_name, devlink);
9867         if (!table)
9868                 goto unlock;
9869         list_del_rcu(&table->list);
9870         mutex_unlock(&devlink->lock);
9871         kfree_rcu(table, rcu);
9872         return;
9873 unlock:
9874         mutex_unlock(&devlink->lock);
9875 }
9876 EXPORT_SYMBOL_GPL(devlink_dpipe_table_unregister);
9877
9878 /**
9879  *      devlink_resource_register - devlink resource register
9880  *
9881  *      @devlink: devlink
9882  *      @resource_name: resource's name
9883  *      @resource_size: resource's size
9884  *      @resource_id: resource's id
9885  *      @parent_resource_id: resource's parent id
9886  *      @size_params: size parameters
9887  *
9888  *      Generic resources should reuse the same names across drivers.
9889  *      Please see the generic resources list at:
9890  *      Documentation/networking/devlink/devlink-resource.rst
9891  */
9892 int devlink_resource_register(struct devlink *devlink,
9893                               const char *resource_name,
9894                               u64 resource_size,
9895                               u64 resource_id,
9896                               u64 parent_resource_id,
9897                               const struct devlink_resource_size_params *size_params)
9898 {
9899         struct devlink_resource *resource;
9900         struct list_head *resource_list;
9901         bool top_hierarchy;
9902         int err = 0;
9903
9904         top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP;
9905
9906         mutex_lock(&devlink->lock);
9907         resource = devlink_resource_find(devlink, NULL, resource_id);
9908         if (resource) {
9909                 err = -EINVAL;
9910                 goto out;
9911         }
9912
9913         resource = kzalloc(sizeof(*resource), GFP_KERNEL);
9914         if (!resource) {
9915                 err = -ENOMEM;
9916                 goto out;
9917         }
9918
9919         if (top_hierarchy) {
9920                 resource_list = &devlink->resource_list;
9921         } else {
9922                 struct devlink_resource *parent_resource;
9923
9924                 parent_resource = devlink_resource_find(devlink, NULL,
9925                                                         parent_resource_id);
9926                 if (parent_resource) {
9927                         resource_list = &parent_resource->resource_list;
9928                         resource->parent = parent_resource;
9929                 } else {
9930                         kfree(resource);
9931                         err = -EINVAL;
9932                         goto out;
9933                 }
9934         }
9935
9936         resource->name = resource_name;
9937         resource->size = resource_size;
9938         resource->size_new = resource_size;
9939         resource->id = resource_id;
9940         resource->size_valid = true;
9941         memcpy(&resource->size_params, size_params,
9942                sizeof(resource->size_params));
9943         INIT_LIST_HEAD(&resource->resource_list);
9944         list_add_tail(&resource->list, resource_list);
9945 out:
9946         mutex_unlock(&devlink->lock);
9947         return err;
9948 }
9949 EXPORT_SYMBOL_GPL(devlink_resource_register);
9950
9951 static void devlink_resource_unregister(struct devlink *devlink,
9952                                         struct devlink_resource *resource)
9953 {
9954         struct devlink_resource *tmp, *child_resource;
9955
9956         list_for_each_entry_safe(child_resource, tmp, &resource->resource_list,
9957                                  list) {
9958                 devlink_resource_unregister(devlink, child_resource);
9959                 list_del(&child_resource->list);
9960                 kfree(child_resource);
9961         }
9962 }
9963
9964 /**
9965  *      devlink_resources_unregister - free all resources
9966  *
9967  *      @devlink: devlink
9968  */
9969 void devlink_resources_unregister(struct devlink *devlink)
9970 {
9971         struct devlink_resource *tmp, *child_resource;
9972
9973         mutex_lock(&devlink->lock);
9974
9975         list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list,
9976                                  list) {
9977                 devlink_resource_unregister(devlink, child_resource);
9978                 list_del(&child_resource->list);
9979                 kfree(child_resource);
9980         }
9981
9982         mutex_unlock(&devlink->lock);
9983 }
9984 EXPORT_SYMBOL_GPL(devlink_resources_unregister);
9985
9986 /**
9987  *      devlink_resource_size_get - get and update size
9988  *
9989  *      @devlink: devlink
9990  *      @resource_id: the requested resource id
9991  *      @p_resource_size: ptr to update
9992  */
9993 int devlink_resource_size_get(struct devlink *devlink,
9994                               u64 resource_id,
9995                               u64 *p_resource_size)
9996 {
9997         struct devlink_resource *resource;
9998         int err = 0;
9999
10000         mutex_lock(&devlink->lock);
10001         resource = devlink_resource_find(devlink, NULL, resource_id);
10002         if (!resource) {
10003                 err = -EINVAL;
10004                 goto out;
10005         }
10006         *p_resource_size = resource->size_new;
10007         resource->size = resource->size_new;
10008 out:
10009         mutex_unlock(&devlink->lock);
10010         return err;
10011 }
10012 EXPORT_SYMBOL_GPL(devlink_resource_size_get);
10013
10014 /**
10015  *      devlink_dpipe_table_resource_set - set the resource id
10016  *
10017  *      @devlink: devlink
10018  *      @table_name: table name
10019  *      @resource_id: resource id
10020  *      @resource_units: number of resource's units consumed per table's entry
10021  */
10022 int devlink_dpipe_table_resource_set(struct devlink *devlink,
10023                                      const char *table_name, u64 resource_id,
10024                                      u64 resource_units)
10025 {
10026         struct devlink_dpipe_table *table;
10027         int err = 0;
10028
10029         mutex_lock(&devlink->lock);
10030         table = devlink_dpipe_table_find(&devlink->dpipe_table_list,
10031                                          table_name, devlink);
10032         if (!table) {
10033                 err = -EINVAL;
10034                 goto out;
10035         }
10036         table->resource_id = resource_id;
10037         table->resource_units = resource_units;
10038         table->resource_valid = true;
10039 out:
10040         mutex_unlock(&devlink->lock);
10041         return err;
10042 }
10043 EXPORT_SYMBOL_GPL(devlink_dpipe_table_resource_set);
10044
10045 /**
10046  *      devlink_resource_occ_get_register - register occupancy getter
10047  *
10048  *      @devlink: devlink
10049  *      @resource_id: resource id
10050  *      @occ_get: occupancy getter callback
10051  *      @occ_get_priv: occupancy getter callback priv
10052  */
10053 void devlink_resource_occ_get_register(struct devlink *devlink,
10054                                        u64 resource_id,
10055                                        devlink_resource_occ_get_t *occ_get,
10056                                        void *occ_get_priv)
10057 {
10058         struct devlink_resource *resource;
10059
10060         mutex_lock(&devlink->lock);
10061         resource = devlink_resource_find(devlink, NULL, resource_id);
10062         if (WARN_ON(!resource))
10063                 goto out;
10064         WARN_ON(resource->occ_get);
10065
10066         resource->occ_get = occ_get;
10067         resource->occ_get_priv = occ_get_priv;
10068 out:
10069         mutex_unlock(&devlink->lock);
10070 }
10071 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_register);
10072
10073 /**
10074  *      devlink_resource_occ_get_unregister - unregister occupancy getter
10075  *
10076  *      @devlink: devlink
10077  *      @resource_id: resource id
10078  */
10079 void devlink_resource_occ_get_unregister(struct devlink *devlink,
10080                                          u64 resource_id)
10081 {
10082         struct devlink_resource *resource;
10083
10084         mutex_lock(&devlink->lock);
10085         resource = devlink_resource_find(devlink, NULL, resource_id);
10086         if (WARN_ON(!resource))
10087                 goto out;
10088         WARN_ON(!resource->occ_get);
10089
10090         resource->occ_get = NULL;
10091         resource->occ_get_priv = NULL;
10092 out:
10093         mutex_unlock(&devlink->lock);
10094 }
10095 EXPORT_SYMBOL_GPL(devlink_resource_occ_get_unregister);
10096
10097 static int devlink_param_verify(const struct devlink_param *param)
10098 {
10099         if (!param || !param->name || !param->supported_cmodes)
10100                 return -EINVAL;
10101         if (param->generic)
10102                 return devlink_param_generic_verify(param);
10103         else
10104                 return devlink_param_driver_verify(param);
10105 }
10106
10107 /**
10108  *      devlink_params_register - register configuration parameters
10109  *
10110  *      @devlink: devlink
10111  *      @params: configuration parameters array
10112  *      @params_count: number of parameters provided
10113  *
10114  *      Register the configuration parameters supported by the driver.
10115  */
10116 int devlink_params_register(struct devlink *devlink,
10117                             const struct devlink_param *params,
10118                             size_t params_count)
10119 {
10120         const struct devlink_param *param = params;
10121         int i, err;
10122
10123         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10124
10125         for (i = 0; i < params_count; i++, param++) {
10126                 err = devlink_param_register(devlink, param);
10127                 if (err)
10128                         goto rollback;
10129         }
10130         return 0;
10131
10132 rollback:
10133         if (!i)
10134                 return err;
10135
10136         for (param--; i > 0; i--, param--)
10137                 devlink_param_unregister(devlink, param);
10138         return err;
10139 }
10140 EXPORT_SYMBOL_GPL(devlink_params_register);
10141
10142 /**
10143  *      devlink_params_unregister - unregister configuration parameters
10144  *      @devlink: devlink
10145  *      @params: configuration parameters to unregister
10146  *      @params_count: number of parameters provided
10147  */
10148 void devlink_params_unregister(struct devlink *devlink,
10149                                const struct devlink_param *params,
10150                                size_t params_count)
10151 {
10152         const struct devlink_param *param = params;
10153         int i;
10154
10155         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10156
10157         for (i = 0; i < params_count; i++, param++)
10158                 devlink_param_unregister(devlink, param);
10159 }
10160 EXPORT_SYMBOL_GPL(devlink_params_unregister);
10161
10162 /**
10163  * devlink_param_register - register one configuration parameter
10164  *
10165  * @devlink: devlink
10166  * @param: one configuration parameter
10167  *
10168  * Register the configuration parameter supported by the driver.
10169  * Return: returns 0 on successful registration or error code otherwise.
10170  */
10171 int devlink_param_register(struct devlink *devlink,
10172                            const struct devlink_param *param)
10173 {
10174         struct devlink_param_item *param_item;
10175
10176         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10177
10178         WARN_ON(devlink_param_verify(param));
10179         WARN_ON(devlink_param_find_by_name(&devlink->param_list, param->name));
10180
10181         if (param->supported_cmodes == BIT(DEVLINK_PARAM_CMODE_DRIVERINIT))
10182                 WARN_ON(param->get || param->set);
10183         else
10184                 WARN_ON(!param->get || !param->set);
10185
10186         param_item = kzalloc(sizeof(*param_item), GFP_KERNEL);
10187         if (!param_item)
10188                 return -ENOMEM;
10189
10190         param_item->param = param;
10191
10192         list_add_tail(&param_item->list, &devlink->param_list);
10193         return 0;
10194 }
10195 EXPORT_SYMBOL_GPL(devlink_param_register);
10196
10197 /**
10198  * devlink_param_unregister - unregister one configuration parameter
10199  * @devlink: devlink
10200  * @param: configuration parameter to unregister
10201  */
10202 void devlink_param_unregister(struct devlink *devlink,
10203                               const struct devlink_param *param)
10204 {
10205         struct devlink_param_item *param_item;
10206
10207         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10208
10209         param_item =
10210                 devlink_param_find_by_name(&devlink->param_list, param->name);
10211         WARN_ON(!param_item);
10212         list_del(&param_item->list);
10213         kfree(param_item);
10214 }
10215 EXPORT_SYMBOL_GPL(devlink_param_unregister);
10216
10217 /**
10218  *      devlink_param_driverinit_value_get - get configuration parameter
10219  *                                           value for driver initializing
10220  *
10221  *      @devlink: devlink
10222  *      @param_id: parameter ID
10223  *      @init_val: value of parameter in driverinit configuration mode
10224  *
10225  *      This function should be used by the driver to get driverinit
10226  *      configuration for initialization after reload command.
10227  */
10228 int devlink_param_driverinit_value_get(struct devlink *devlink, u32 param_id,
10229                                        union devlink_param_value *init_val)
10230 {
10231         struct devlink_param_item *param_item;
10232
10233         if (!devlink_reload_supported(devlink->ops))
10234                 return -EOPNOTSUPP;
10235
10236         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10237         if (!param_item)
10238                 return -EINVAL;
10239
10240         if (!param_item->driverinit_value_valid ||
10241             !devlink_param_cmode_is_supported(param_item->param,
10242                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10243                 return -EOPNOTSUPP;
10244
10245         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10246                 strcpy(init_val->vstr, param_item->driverinit_value.vstr);
10247         else
10248                 *init_val = param_item->driverinit_value;
10249
10250         return 0;
10251 }
10252 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_get);
10253
10254 /**
10255  *      devlink_param_driverinit_value_set - set value of configuration
10256  *                                           parameter for driverinit
10257  *                                           configuration mode
10258  *
10259  *      @devlink: devlink
10260  *      @param_id: parameter ID
10261  *      @init_val: value of parameter to set for driverinit configuration mode
10262  *
10263  *      This function should be used by the driver to set driverinit
10264  *      configuration mode default value.
10265  */
10266 int devlink_param_driverinit_value_set(struct devlink *devlink, u32 param_id,
10267                                        union devlink_param_value init_val)
10268 {
10269         struct devlink_param_item *param_item;
10270
10271         ASSERT_DEVLINK_NOT_REGISTERED(devlink);
10272
10273         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10274         if (!param_item)
10275                 return -EINVAL;
10276
10277         if (!devlink_param_cmode_is_supported(param_item->param,
10278                                               DEVLINK_PARAM_CMODE_DRIVERINIT))
10279                 return -EOPNOTSUPP;
10280
10281         if (param_item->param->type == DEVLINK_PARAM_TYPE_STRING)
10282                 strcpy(param_item->driverinit_value.vstr, init_val.vstr);
10283         else
10284                 param_item->driverinit_value = init_val;
10285         param_item->driverinit_value_valid = true;
10286         return 0;
10287 }
10288 EXPORT_SYMBOL_GPL(devlink_param_driverinit_value_set);
10289
10290 /**
10291  *      devlink_param_value_changed - notify devlink on a parameter's value
10292  *                                    change. Should be called by the driver
10293  *                                    right after the change.
10294  *
10295  *      @devlink: devlink
10296  *      @param_id: parameter ID
10297  *
10298  *      This function should be used by the driver to notify devlink on value
10299  *      change, excluding driverinit configuration mode.
10300  *      For driverinit configuration mode driver should use the function
10301  */
10302 void devlink_param_value_changed(struct devlink *devlink, u32 param_id)
10303 {
10304         struct devlink_param_item *param_item;
10305
10306         param_item = devlink_param_find_by_id(&devlink->param_list, param_id);
10307         WARN_ON(!param_item);
10308
10309         devlink_param_notify(devlink, 0, param_item, DEVLINK_CMD_PARAM_NEW);
10310 }
10311 EXPORT_SYMBOL_GPL(devlink_param_value_changed);
10312
10313 /**
10314  *      devlink_region_create - create a new address region
10315  *
10316  *      @devlink: devlink
10317  *      @ops: region operations and name
10318  *      @region_max_snapshots: Maximum supported number of snapshots for region
10319  *      @region_size: size of region
10320  */
10321 struct devlink_region *
10322 devlink_region_create(struct devlink *devlink,
10323                       const struct devlink_region_ops *ops,
10324                       u32 region_max_snapshots, u64 region_size)
10325 {
10326         struct devlink_region *region;
10327         int err = 0;
10328
10329         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
10330                 return ERR_PTR(-EINVAL);
10331
10332         mutex_lock(&devlink->lock);
10333
10334         if (devlink_region_get_by_name(devlink, ops->name)) {
10335                 err = -EEXIST;
10336                 goto unlock;
10337         }
10338
10339         region = kzalloc(sizeof(*region), GFP_KERNEL);
10340         if (!region) {
10341                 err = -ENOMEM;
10342                 goto unlock;
10343         }
10344
10345         region->devlink = devlink;
10346         region->max_snapshots = region_max_snapshots;
10347         region->ops = ops;
10348         region->size = region_size;
10349         INIT_LIST_HEAD(&region->snapshot_list);
10350         list_add_tail(&region->list, &devlink->region_list);
10351         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
10352
10353         mutex_unlock(&devlink->lock);
10354         return region;
10355
10356 unlock:
10357         mutex_unlock(&devlink->lock);
10358         return ERR_PTR(err);
10359 }
10360 EXPORT_SYMBOL_GPL(devlink_region_create);
10361
10362 /**
10363  *      devlink_port_region_create - create a new address region for a port
10364  *
10365  *      @port: devlink port
10366  *      @ops: region operations and name
10367  *      @region_max_snapshots: Maximum supported number of snapshots for region
10368  *      @region_size: size of region
10369  */
10370 struct devlink_region *
10371 devlink_port_region_create(struct devlink_port *port,
10372                            const struct devlink_port_region_ops *ops,
10373                            u32 region_max_snapshots, u64 region_size)
10374 {
10375         struct devlink *devlink = port->devlink;
10376         struct devlink_region *region;
10377         int err = 0;
10378
10379         if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
10380                 return ERR_PTR(-EINVAL);
10381
10382         mutex_lock(&devlink->lock);
10383
10384         if (devlink_port_region_get_by_name(port, ops->name)) {
10385                 err = -EEXIST;
10386                 goto unlock;
10387         }
10388
10389         region = kzalloc(sizeof(*region), GFP_KERNEL);
10390         if (!region) {
10391                 err = -ENOMEM;
10392                 goto unlock;
10393         }
10394
10395         region->devlink = devlink;
10396         region->port = port;
10397         region->max_snapshots = region_max_snapshots;
10398         region->port_ops = ops;
10399         region->size = region_size;
10400         INIT_LIST_HEAD(&region->snapshot_list);
10401         list_add_tail(&region->list, &port->region_list);
10402         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_NEW);
10403
10404         mutex_unlock(&devlink->lock);
10405         return region;
10406
10407 unlock:
10408         mutex_unlock(&devlink->lock);
10409         return ERR_PTR(err);
10410 }
10411 EXPORT_SYMBOL_GPL(devlink_port_region_create);
10412
10413 /**
10414  *      devlink_region_destroy - destroy address region
10415  *
10416  *      @region: devlink region to destroy
10417  */
10418 void devlink_region_destroy(struct devlink_region *region)
10419 {
10420         struct devlink *devlink = region->devlink;
10421         struct devlink_snapshot *snapshot, *ts;
10422
10423         mutex_lock(&devlink->lock);
10424
10425         /* Free all snapshots of region */
10426         list_for_each_entry_safe(snapshot, ts, &region->snapshot_list, list)
10427                 devlink_region_snapshot_del(region, snapshot);
10428
10429         list_del(&region->list);
10430
10431         devlink_nl_region_notify(region, NULL, DEVLINK_CMD_REGION_DEL);
10432         mutex_unlock(&devlink->lock);
10433         kfree(region);
10434 }
10435 EXPORT_SYMBOL_GPL(devlink_region_destroy);
10436
10437 /**
10438  *      devlink_region_snapshot_id_get - get snapshot ID
10439  *
10440  *      This callback should be called when adding a new snapshot,
10441  *      Driver should use the same id for multiple snapshots taken
10442  *      on multiple regions at the same time/by the same trigger.
10443  *
10444  *      The caller of this function must use devlink_region_snapshot_id_put
10445  *      when finished creating regions using this id.
10446  *
10447  *      Returns zero on success, or a negative error code on failure.
10448  *
10449  *      @devlink: devlink
10450  *      @id: storage to return id
10451  */
10452 int devlink_region_snapshot_id_get(struct devlink *devlink, u32 *id)
10453 {
10454         int err;
10455
10456         mutex_lock(&devlink->lock);
10457         err = __devlink_region_snapshot_id_get(devlink, id);
10458         mutex_unlock(&devlink->lock);
10459
10460         return err;
10461 }
10462 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_get);
10463
10464 /**
10465  *      devlink_region_snapshot_id_put - put snapshot ID reference
10466  *
10467  *      This should be called by a driver after finishing creating snapshots
10468  *      with an id. Doing so ensures that the ID can later be released in the
10469  *      event that all snapshots using it have been destroyed.
10470  *
10471  *      @devlink: devlink
10472  *      @id: id to release reference on
10473  */
10474 void devlink_region_snapshot_id_put(struct devlink *devlink, u32 id)
10475 {
10476         mutex_lock(&devlink->lock);
10477         __devlink_snapshot_id_decrement(devlink, id);
10478         mutex_unlock(&devlink->lock);
10479 }
10480 EXPORT_SYMBOL_GPL(devlink_region_snapshot_id_put);
10481
10482 /**
10483  *      devlink_region_snapshot_create - create a new snapshot
10484  *      This will add a new snapshot of a region. The snapshot
10485  *      will be stored on the region struct and can be accessed
10486  *      from devlink. This is useful for future analyses of snapshots.
10487  *      Multiple snapshots can be created on a region.
10488  *      The @snapshot_id should be obtained using the getter function.
10489  *
10490  *      @region: devlink region of the snapshot
10491  *      @data: snapshot data
10492  *      @snapshot_id: snapshot id to be created
10493  */
10494 int devlink_region_snapshot_create(struct devlink_region *region,
10495                                    u8 *data, u32 snapshot_id)
10496 {
10497         struct devlink *devlink = region->devlink;
10498         int err;
10499
10500         mutex_lock(&devlink->lock);
10501         err = __devlink_region_snapshot_create(region, data, snapshot_id);
10502         mutex_unlock(&devlink->lock);
10503
10504         return err;
10505 }
10506 EXPORT_SYMBOL_GPL(devlink_region_snapshot_create);
10507
10508 #define DEVLINK_TRAP(_id, _type)                                              \
10509         {                                                                     \
10510                 .type = DEVLINK_TRAP_TYPE_##_type,                            \
10511                 .id = DEVLINK_TRAP_GENERIC_ID_##_id,                          \
10512                 .name = DEVLINK_TRAP_GENERIC_NAME_##_id,                      \
10513         }
10514
10515 static const struct devlink_trap devlink_trap_generic[] = {
10516         DEVLINK_TRAP(SMAC_MC, DROP),
10517         DEVLINK_TRAP(VLAN_TAG_MISMATCH, DROP),
10518         DEVLINK_TRAP(INGRESS_VLAN_FILTER, DROP),
10519         DEVLINK_TRAP(INGRESS_STP_FILTER, DROP),
10520         DEVLINK_TRAP(EMPTY_TX_LIST, DROP),
10521         DEVLINK_TRAP(PORT_LOOPBACK_FILTER, DROP),
10522         DEVLINK_TRAP(BLACKHOLE_ROUTE, DROP),
10523         DEVLINK_TRAP(TTL_ERROR, EXCEPTION),
10524         DEVLINK_TRAP(TAIL_DROP, DROP),
10525         DEVLINK_TRAP(NON_IP_PACKET, DROP),
10526         DEVLINK_TRAP(UC_DIP_MC_DMAC, DROP),
10527         DEVLINK_TRAP(DIP_LB, DROP),
10528         DEVLINK_TRAP(SIP_MC, DROP),
10529         DEVLINK_TRAP(SIP_LB, DROP),
10530         DEVLINK_TRAP(CORRUPTED_IP_HDR, DROP),
10531         DEVLINK_TRAP(IPV4_SIP_BC, DROP),
10532         DEVLINK_TRAP(IPV6_MC_DIP_RESERVED_SCOPE, DROP),
10533         DEVLINK_TRAP(IPV6_MC_DIP_INTERFACE_LOCAL_SCOPE, DROP),
10534         DEVLINK_TRAP(MTU_ERROR, EXCEPTION),
10535         DEVLINK_TRAP(UNRESOLVED_NEIGH, EXCEPTION),
10536         DEVLINK_TRAP(RPF, EXCEPTION),
10537         DEVLINK_TRAP(REJECT_ROUTE, EXCEPTION),
10538         DEVLINK_TRAP(IPV4_LPM_UNICAST_MISS, EXCEPTION),
10539         DEVLINK_TRAP(IPV6_LPM_UNICAST_MISS, EXCEPTION),
10540         DEVLINK_TRAP(NON_ROUTABLE, DROP),
10541         DEVLINK_TRAP(DECAP_ERROR, EXCEPTION),
10542         DEVLINK_TRAP(OVERLAY_SMAC_MC, DROP),
10543         DEVLINK_TRAP(INGRESS_FLOW_ACTION_DROP, DROP),
10544         DEVLINK_TRAP(EGRESS_FLOW_ACTION_DROP, DROP),
10545         DEVLINK_TRAP(STP, CONTROL),
10546         DEVLINK_TRAP(LACP, CONTROL),
10547         DEVLINK_TRAP(LLDP, CONTROL),
10548         DEVLINK_TRAP(IGMP_QUERY, CONTROL),
10549         DEVLINK_TRAP(IGMP_V1_REPORT, CONTROL),
10550         DEVLINK_TRAP(IGMP_V2_REPORT, CONTROL),
10551         DEVLINK_TRAP(IGMP_V3_REPORT, CONTROL),
10552         DEVLINK_TRAP(IGMP_V2_LEAVE, CONTROL),
10553         DEVLINK_TRAP(MLD_QUERY, CONTROL),
10554         DEVLINK_TRAP(MLD_V1_REPORT, CONTROL),
10555         DEVLINK_TRAP(MLD_V2_REPORT, CONTROL),
10556         DEVLINK_TRAP(MLD_V1_DONE, CONTROL),
10557         DEVLINK_TRAP(IPV4_DHCP, CONTROL),
10558         DEVLINK_TRAP(IPV6_DHCP, CONTROL),
10559         DEVLINK_TRAP(ARP_REQUEST, CONTROL),
10560         DEVLINK_TRAP(ARP_RESPONSE, CONTROL),
10561         DEVLINK_TRAP(ARP_OVERLAY, CONTROL),
10562         DEVLINK_TRAP(IPV6_NEIGH_SOLICIT, CONTROL),
10563         DEVLINK_TRAP(IPV6_NEIGH_ADVERT, CONTROL),
10564         DEVLINK_TRAP(IPV4_BFD, CONTROL),
10565         DEVLINK_TRAP(IPV6_BFD, CONTROL),
10566         DEVLINK_TRAP(IPV4_OSPF, CONTROL),
10567         DEVLINK_TRAP(IPV6_OSPF, CONTROL),
10568         DEVLINK_TRAP(IPV4_BGP, CONTROL),
10569         DEVLINK_TRAP(IPV6_BGP, CONTROL),
10570         DEVLINK_TRAP(IPV4_VRRP, CONTROL),
10571         DEVLINK_TRAP(IPV6_VRRP, CONTROL),
10572         DEVLINK_TRAP(IPV4_PIM, CONTROL),
10573         DEVLINK_TRAP(IPV6_PIM, CONTROL),
10574         DEVLINK_TRAP(UC_LB, CONTROL),
10575         DEVLINK_TRAP(LOCAL_ROUTE, CONTROL),
10576         DEVLINK_TRAP(EXTERNAL_ROUTE, CONTROL),
10577         DEVLINK_TRAP(IPV6_UC_DIP_LINK_LOCAL_SCOPE, CONTROL),
10578         DEVLINK_TRAP(IPV6_DIP_ALL_NODES, CONTROL),
10579         DEVLINK_TRAP(IPV6_DIP_ALL_ROUTERS, CONTROL),
10580         DEVLINK_TRAP(IPV6_ROUTER_SOLICIT, CONTROL),
10581         DEVLINK_TRAP(IPV6_ROUTER_ADVERT, CONTROL),
10582         DEVLINK_TRAP(IPV6_REDIRECT, CONTROL),
10583         DEVLINK_TRAP(IPV4_ROUTER_ALERT, CONTROL),
10584         DEVLINK_TRAP(IPV6_ROUTER_ALERT, CONTROL),
10585         DEVLINK_TRAP(PTP_EVENT, CONTROL),
10586         DEVLINK_TRAP(PTP_GENERAL, CONTROL),
10587         DEVLINK_TRAP(FLOW_ACTION_SAMPLE, CONTROL),
10588         DEVLINK_TRAP(FLOW_ACTION_TRAP, CONTROL),
10589         DEVLINK_TRAP(EARLY_DROP, DROP),
10590         DEVLINK_TRAP(VXLAN_PARSING, DROP),
10591         DEVLINK_TRAP(LLC_SNAP_PARSING, DROP),
10592         DEVLINK_TRAP(VLAN_PARSING, DROP),
10593         DEVLINK_TRAP(PPPOE_PPP_PARSING, DROP),
10594         DEVLINK_TRAP(MPLS_PARSING, DROP),
10595         DEVLINK_TRAP(ARP_PARSING, DROP),
10596         DEVLINK_TRAP(IP_1_PARSING, DROP),
10597         DEVLINK_TRAP(IP_N_PARSING, DROP),
10598         DEVLINK_TRAP(GRE_PARSING, DROP),
10599         DEVLINK_TRAP(UDP_PARSING, DROP),
10600         DEVLINK_TRAP(TCP_PARSING, DROP),
10601         DEVLINK_TRAP(IPSEC_PARSING, DROP),
10602         DEVLINK_TRAP(SCTP_PARSING, DROP),
10603         DEVLINK_TRAP(DCCP_PARSING, DROP),
10604         DEVLINK_TRAP(GTP_PARSING, DROP),
10605         DEVLINK_TRAP(ESP_PARSING, DROP),
10606         DEVLINK_TRAP(BLACKHOLE_NEXTHOP, DROP),
10607         DEVLINK_TRAP(DMAC_FILTER, DROP),
10608 };
10609
10610 #define DEVLINK_TRAP_GROUP(_id)                                               \
10611         {                                                                     \
10612                 .id = DEVLINK_TRAP_GROUP_GENERIC_ID_##_id,                    \
10613                 .name = DEVLINK_TRAP_GROUP_GENERIC_NAME_##_id,                \
10614         }
10615
10616 static const struct devlink_trap_group devlink_trap_group_generic[] = {
10617         DEVLINK_TRAP_GROUP(L2_DROPS),
10618         DEVLINK_TRAP_GROUP(L3_DROPS),
10619         DEVLINK_TRAP_GROUP(L3_EXCEPTIONS),
10620         DEVLINK_TRAP_GROUP(BUFFER_DROPS),
10621         DEVLINK_TRAP_GROUP(TUNNEL_DROPS),
10622         DEVLINK_TRAP_GROUP(ACL_DROPS),
10623         DEVLINK_TRAP_GROUP(STP),
10624         DEVLINK_TRAP_GROUP(LACP),
10625         DEVLINK_TRAP_GROUP(LLDP),
10626         DEVLINK_TRAP_GROUP(MC_SNOOPING),
10627         DEVLINK_TRAP_GROUP(DHCP),
10628         DEVLINK_TRAP_GROUP(NEIGH_DISCOVERY),
10629         DEVLINK_TRAP_GROUP(BFD),
10630         DEVLINK_TRAP_GROUP(OSPF),
10631         DEVLINK_TRAP_GROUP(BGP),
10632         DEVLINK_TRAP_GROUP(VRRP),
10633         DEVLINK_TRAP_GROUP(PIM),
10634         DEVLINK_TRAP_GROUP(UC_LB),
10635         DEVLINK_TRAP_GROUP(LOCAL_DELIVERY),
10636         DEVLINK_TRAP_GROUP(EXTERNAL_DELIVERY),
10637         DEVLINK_TRAP_GROUP(IPV6),
10638         DEVLINK_TRAP_GROUP(PTP_EVENT),
10639         DEVLINK_TRAP_GROUP(PTP_GENERAL),
10640         DEVLINK_TRAP_GROUP(ACL_SAMPLE),
10641         DEVLINK_TRAP_GROUP(ACL_TRAP),
10642         DEVLINK_TRAP_GROUP(PARSER_ERROR_DROPS),
10643 };
10644
10645 static int devlink_trap_generic_verify(const struct devlink_trap *trap)
10646 {
10647         if (trap->id > DEVLINK_TRAP_GENERIC_ID_MAX)
10648                 return -EINVAL;
10649
10650         if (strcmp(trap->name, devlink_trap_generic[trap->id].name))
10651                 return -EINVAL;
10652
10653         if (trap->type != devlink_trap_generic[trap->id].type)
10654                 return -EINVAL;
10655
10656         return 0;
10657 }
10658
10659 static int devlink_trap_driver_verify(const struct devlink_trap *trap)
10660 {
10661         int i;
10662
10663         if (trap->id <= DEVLINK_TRAP_GENERIC_ID_MAX)
10664                 return -EINVAL;
10665
10666         for (i = 0; i < ARRAY_SIZE(devlink_trap_generic); i++) {
10667                 if (!strcmp(trap->name, devlink_trap_generic[i].name))
10668                         return -EEXIST;
10669         }
10670
10671         return 0;
10672 }
10673
10674 static int devlink_trap_verify(const struct devlink_trap *trap)
10675 {
10676         if (!trap || !trap->name)
10677                 return -EINVAL;
10678
10679         if (trap->generic)
10680                 return devlink_trap_generic_verify(trap);
10681         else
10682                 return devlink_trap_driver_verify(trap);
10683 }
10684
10685 static int
10686 devlink_trap_group_generic_verify(const struct devlink_trap_group *group)
10687 {
10688         if (group->id > DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
10689                 return -EINVAL;
10690
10691         if (strcmp(group->name, devlink_trap_group_generic[group->id].name))
10692                 return -EINVAL;
10693
10694         return 0;
10695 }
10696
10697 static int
10698 devlink_trap_group_driver_verify(const struct devlink_trap_group *group)
10699 {
10700         int i;
10701
10702         if (group->id <= DEVLINK_TRAP_GROUP_GENERIC_ID_MAX)
10703                 return -EINVAL;
10704
10705         for (i = 0; i < ARRAY_SIZE(devlink_trap_group_generic); i++) {
10706                 if (!strcmp(group->name, devlink_trap_group_generic[i].name))
10707                         return -EEXIST;
10708         }
10709
10710         return 0;
10711 }
10712
10713 static int devlink_trap_group_verify(const struct devlink_trap_group *group)
10714 {
10715         if (group->generic)
10716                 return devlink_trap_group_generic_verify(group);
10717         else
10718                 return devlink_trap_group_driver_verify(group);
10719 }
10720
10721 static void
10722 devlink_trap_group_notify(struct devlink *devlink,
10723                           const struct devlink_trap_group_item *group_item,
10724                           enum devlink_command cmd)
10725 {
10726         struct sk_buff *msg;
10727         int err;
10728
10729         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_GROUP_NEW &&
10730                      cmd != DEVLINK_CMD_TRAP_GROUP_DEL);
10731         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
10732                 return;
10733
10734         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10735         if (!msg)
10736                 return;
10737
10738         err = devlink_nl_trap_group_fill(msg, devlink, group_item, cmd, 0, 0,
10739                                          0);
10740         if (err) {
10741                 nlmsg_free(msg);
10742                 return;
10743         }
10744
10745         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
10746                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
10747 }
10748
10749 static int
10750 devlink_trap_item_group_link(struct devlink *devlink,
10751                              struct devlink_trap_item *trap_item)
10752 {
10753         u16 group_id = trap_item->trap->init_group_id;
10754         struct devlink_trap_group_item *group_item;
10755
10756         group_item = devlink_trap_group_item_lookup_by_id(devlink, group_id);
10757         if (WARN_ON_ONCE(!group_item))
10758                 return -EINVAL;
10759
10760         trap_item->group_item = group_item;
10761
10762         return 0;
10763 }
10764
10765 static void devlink_trap_notify(struct devlink *devlink,
10766                                 const struct devlink_trap_item *trap_item,
10767                                 enum devlink_command cmd)
10768 {
10769         struct sk_buff *msg;
10770         int err;
10771
10772         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_NEW &&
10773                      cmd != DEVLINK_CMD_TRAP_DEL);
10774         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
10775                 return;
10776
10777         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
10778         if (!msg)
10779                 return;
10780
10781         err = devlink_nl_trap_fill(msg, devlink, trap_item, cmd, 0, 0, 0);
10782         if (err) {
10783                 nlmsg_free(msg);
10784                 return;
10785         }
10786
10787         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
10788                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
10789 }
10790
10791 static int
10792 devlink_trap_register(struct devlink *devlink,
10793                       const struct devlink_trap *trap, void *priv)
10794 {
10795         struct devlink_trap_item *trap_item;
10796         int err;
10797
10798         if (devlink_trap_item_lookup(devlink, trap->name))
10799                 return -EEXIST;
10800
10801         trap_item = kzalloc(sizeof(*trap_item), GFP_KERNEL);
10802         if (!trap_item)
10803                 return -ENOMEM;
10804
10805         trap_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
10806         if (!trap_item->stats) {
10807                 err = -ENOMEM;
10808                 goto err_stats_alloc;
10809         }
10810
10811         trap_item->trap = trap;
10812         trap_item->action = trap->init_action;
10813         trap_item->priv = priv;
10814
10815         err = devlink_trap_item_group_link(devlink, trap_item);
10816         if (err)
10817                 goto err_group_link;
10818
10819         err = devlink->ops->trap_init(devlink, trap, trap_item);
10820         if (err)
10821                 goto err_trap_init;
10822
10823         list_add_tail(&trap_item->list, &devlink->trap_list);
10824         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_NEW);
10825
10826         return 0;
10827
10828 err_trap_init:
10829 err_group_link:
10830         free_percpu(trap_item->stats);
10831 err_stats_alloc:
10832         kfree(trap_item);
10833         return err;
10834 }
10835
10836 static void devlink_trap_unregister(struct devlink *devlink,
10837                                     const struct devlink_trap *trap)
10838 {
10839         struct devlink_trap_item *trap_item;
10840
10841         trap_item = devlink_trap_item_lookup(devlink, trap->name);
10842         if (WARN_ON_ONCE(!trap_item))
10843                 return;
10844
10845         devlink_trap_notify(devlink, trap_item, DEVLINK_CMD_TRAP_DEL);
10846         list_del(&trap_item->list);
10847         if (devlink->ops->trap_fini)
10848                 devlink->ops->trap_fini(devlink, trap, trap_item);
10849         free_percpu(trap_item->stats);
10850         kfree(trap_item);
10851 }
10852
10853 static void devlink_trap_disable(struct devlink *devlink,
10854                                  const struct devlink_trap *trap)
10855 {
10856         struct devlink_trap_item *trap_item;
10857
10858         trap_item = devlink_trap_item_lookup(devlink, trap->name);
10859         if (WARN_ON_ONCE(!trap_item))
10860                 return;
10861
10862         devlink->ops->trap_action_set(devlink, trap, DEVLINK_TRAP_ACTION_DROP,
10863                                       NULL);
10864         trap_item->action = DEVLINK_TRAP_ACTION_DROP;
10865 }
10866
10867 /**
10868  * devlink_traps_register - Register packet traps with devlink.
10869  * @devlink: devlink.
10870  * @traps: Packet traps.
10871  * @traps_count: Count of provided packet traps.
10872  * @priv: Driver private information.
10873  *
10874  * Return: Non-zero value on failure.
10875  */
10876 int devlink_traps_register(struct devlink *devlink,
10877                            const struct devlink_trap *traps,
10878                            size_t traps_count, void *priv)
10879 {
10880         int i, err;
10881
10882         if (!devlink->ops->trap_init || !devlink->ops->trap_action_set)
10883                 return -EINVAL;
10884
10885         mutex_lock(&devlink->lock);
10886         for (i = 0; i < traps_count; i++) {
10887                 const struct devlink_trap *trap = &traps[i];
10888
10889                 err = devlink_trap_verify(trap);
10890                 if (err)
10891                         goto err_trap_verify;
10892
10893                 err = devlink_trap_register(devlink, trap, priv);
10894                 if (err)
10895                         goto err_trap_register;
10896         }
10897         mutex_unlock(&devlink->lock);
10898
10899         return 0;
10900
10901 err_trap_register:
10902 err_trap_verify:
10903         for (i--; i >= 0; i--)
10904                 devlink_trap_unregister(devlink, &traps[i]);
10905         mutex_unlock(&devlink->lock);
10906         return err;
10907 }
10908 EXPORT_SYMBOL_GPL(devlink_traps_register);
10909
10910 /**
10911  * devlink_traps_unregister - Unregister packet traps from devlink.
10912  * @devlink: devlink.
10913  * @traps: Packet traps.
10914  * @traps_count: Count of provided packet traps.
10915  */
10916 void devlink_traps_unregister(struct devlink *devlink,
10917                               const struct devlink_trap *traps,
10918                               size_t traps_count)
10919 {
10920         int i;
10921
10922         mutex_lock(&devlink->lock);
10923         /* Make sure we do not have any packets in-flight while unregistering
10924          * traps by disabling all of them and waiting for a grace period.
10925          */
10926         for (i = traps_count - 1; i >= 0; i--)
10927                 devlink_trap_disable(devlink, &traps[i]);
10928         synchronize_rcu();
10929         for (i = traps_count - 1; i >= 0; i--)
10930                 devlink_trap_unregister(devlink, &traps[i]);
10931         mutex_unlock(&devlink->lock);
10932 }
10933 EXPORT_SYMBOL_GPL(devlink_traps_unregister);
10934
10935 static void
10936 devlink_trap_stats_update(struct devlink_stats __percpu *trap_stats,
10937                           size_t skb_len)
10938 {
10939         struct devlink_stats *stats;
10940
10941         stats = this_cpu_ptr(trap_stats);
10942         u64_stats_update_begin(&stats->syncp);
10943         stats->rx_bytes += skb_len;
10944         stats->rx_packets++;
10945         u64_stats_update_end(&stats->syncp);
10946 }
10947
10948 static void
10949 devlink_trap_report_metadata_set(struct devlink_trap_metadata *metadata,
10950                                  const struct devlink_trap_item *trap_item,
10951                                  struct devlink_port *in_devlink_port,
10952                                  const struct flow_action_cookie *fa_cookie)
10953 {
10954         metadata->trap_name = trap_item->trap->name;
10955         metadata->trap_group_name = trap_item->group_item->group->name;
10956         metadata->fa_cookie = fa_cookie;
10957         metadata->trap_type = trap_item->trap->type;
10958
10959         spin_lock(&in_devlink_port->type_lock);
10960         if (in_devlink_port->type == DEVLINK_PORT_TYPE_ETH)
10961                 metadata->input_dev = in_devlink_port->type_dev;
10962         spin_unlock(&in_devlink_port->type_lock);
10963 }
10964
10965 /**
10966  * devlink_trap_report - Report trapped packet to drop monitor.
10967  * @devlink: devlink.
10968  * @skb: Trapped packet.
10969  * @trap_ctx: Trap context.
10970  * @in_devlink_port: Input devlink port.
10971  * @fa_cookie: Flow action cookie. Could be NULL.
10972  */
10973 void devlink_trap_report(struct devlink *devlink, struct sk_buff *skb,
10974                          void *trap_ctx, struct devlink_port *in_devlink_port,
10975                          const struct flow_action_cookie *fa_cookie)
10976
10977 {
10978         struct devlink_trap_item *trap_item = trap_ctx;
10979
10980         devlink_trap_stats_update(trap_item->stats, skb->len);
10981         devlink_trap_stats_update(trap_item->group_item->stats, skb->len);
10982
10983         if (trace_devlink_trap_report_enabled()) {
10984                 struct devlink_trap_metadata metadata = {};
10985
10986                 devlink_trap_report_metadata_set(&metadata, trap_item,
10987                                                  in_devlink_port, fa_cookie);
10988                 trace_devlink_trap_report(devlink, skb, &metadata);
10989         }
10990 }
10991 EXPORT_SYMBOL_GPL(devlink_trap_report);
10992
10993 /**
10994  * devlink_trap_ctx_priv - Trap context to driver private information.
10995  * @trap_ctx: Trap context.
10996  *
10997  * Return: Driver private information passed during registration.
10998  */
10999 void *devlink_trap_ctx_priv(void *trap_ctx)
11000 {
11001         struct devlink_trap_item *trap_item = trap_ctx;
11002
11003         return trap_item->priv;
11004 }
11005 EXPORT_SYMBOL_GPL(devlink_trap_ctx_priv);
11006
11007 static int
11008 devlink_trap_group_item_policer_link(struct devlink *devlink,
11009                                      struct devlink_trap_group_item *group_item)
11010 {
11011         u32 policer_id = group_item->group->init_policer_id;
11012         struct devlink_trap_policer_item *policer_item;
11013
11014         if (policer_id == 0)
11015                 return 0;
11016
11017         policer_item = devlink_trap_policer_item_lookup(devlink, policer_id);
11018         if (WARN_ON_ONCE(!policer_item))
11019                 return -EINVAL;
11020
11021         group_item->policer_item = policer_item;
11022
11023         return 0;
11024 }
11025
11026 static int
11027 devlink_trap_group_register(struct devlink *devlink,
11028                             const struct devlink_trap_group *group)
11029 {
11030         struct devlink_trap_group_item *group_item;
11031         int err;
11032
11033         if (devlink_trap_group_item_lookup(devlink, group->name))
11034                 return -EEXIST;
11035
11036         group_item = kzalloc(sizeof(*group_item), GFP_KERNEL);
11037         if (!group_item)
11038                 return -ENOMEM;
11039
11040         group_item->stats = netdev_alloc_pcpu_stats(struct devlink_stats);
11041         if (!group_item->stats) {
11042                 err = -ENOMEM;
11043                 goto err_stats_alloc;
11044         }
11045
11046         group_item->group = group;
11047
11048         err = devlink_trap_group_item_policer_link(devlink, group_item);
11049         if (err)
11050                 goto err_policer_link;
11051
11052         if (devlink->ops->trap_group_init) {
11053                 err = devlink->ops->trap_group_init(devlink, group);
11054                 if (err)
11055                         goto err_group_init;
11056         }
11057
11058         list_add_tail(&group_item->list, &devlink->trap_group_list);
11059         devlink_trap_group_notify(devlink, group_item,
11060                                   DEVLINK_CMD_TRAP_GROUP_NEW);
11061
11062         return 0;
11063
11064 err_group_init:
11065 err_policer_link:
11066         free_percpu(group_item->stats);
11067 err_stats_alloc:
11068         kfree(group_item);
11069         return err;
11070 }
11071
11072 static void
11073 devlink_trap_group_unregister(struct devlink *devlink,
11074                               const struct devlink_trap_group *group)
11075 {
11076         struct devlink_trap_group_item *group_item;
11077
11078         group_item = devlink_trap_group_item_lookup(devlink, group->name);
11079         if (WARN_ON_ONCE(!group_item))
11080                 return;
11081
11082         devlink_trap_group_notify(devlink, group_item,
11083                                   DEVLINK_CMD_TRAP_GROUP_DEL);
11084         list_del(&group_item->list);
11085         free_percpu(group_item->stats);
11086         kfree(group_item);
11087 }
11088
11089 /**
11090  * devlink_trap_groups_register - Register packet trap groups with devlink.
11091  * @devlink: devlink.
11092  * @groups: Packet trap groups.
11093  * @groups_count: Count of provided packet trap groups.
11094  *
11095  * Return: Non-zero value on failure.
11096  */
11097 int devlink_trap_groups_register(struct devlink *devlink,
11098                                  const struct devlink_trap_group *groups,
11099                                  size_t groups_count)
11100 {
11101         int i, err;
11102
11103         mutex_lock(&devlink->lock);
11104         for (i = 0; i < groups_count; i++) {
11105                 const struct devlink_trap_group *group = &groups[i];
11106
11107                 err = devlink_trap_group_verify(group);
11108                 if (err)
11109                         goto err_trap_group_verify;
11110
11111                 err = devlink_trap_group_register(devlink, group);
11112                 if (err)
11113                         goto err_trap_group_register;
11114         }
11115         mutex_unlock(&devlink->lock);
11116
11117         return 0;
11118
11119 err_trap_group_register:
11120 err_trap_group_verify:
11121         for (i--; i >= 0; i--)
11122                 devlink_trap_group_unregister(devlink, &groups[i]);
11123         mutex_unlock(&devlink->lock);
11124         return err;
11125 }
11126 EXPORT_SYMBOL_GPL(devlink_trap_groups_register);
11127
11128 /**
11129  * devlink_trap_groups_unregister - Unregister packet trap groups from devlink.
11130  * @devlink: devlink.
11131  * @groups: Packet trap groups.
11132  * @groups_count: Count of provided packet trap groups.
11133  */
11134 void devlink_trap_groups_unregister(struct devlink *devlink,
11135                                     const struct devlink_trap_group *groups,
11136                                     size_t groups_count)
11137 {
11138         int i;
11139
11140         mutex_lock(&devlink->lock);
11141         for (i = groups_count - 1; i >= 0; i--)
11142                 devlink_trap_group_unregister(devlink, &groups[i]);
11143         mutex_unlock(&devlink->lock);
11144 }
11145 EXPORT_SYMBOL_GPL(devlink_trap_groups_unregister);
11146
11147 static void
11148 devlink_trap_policer_notify(struct devlink *devlink,
11149                             const struct devlink_trap_policer_item *policer_item,
11150                             enum devlink_command cmd)
11151 {
11152         struct sk_buff *msg;
11153         int err;
11154
11155         WARN_ON_ONCE(cmd != DEVLINK_CMD_TRAP_POLICER_NEW &&
11156                      cmd != DEVLINK_CMD_TRAP_POLICER_DEL);
11157         if (!xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED))
11158                 return;
11159
11160         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11161         if (!msg)
11162                 return;
11163
11164         err = devlink_nl_trap_policer_fill(msg, devlink, policer_item, cmd, 0,
11165                                            0, 0);
11166         if (err) {
11167                 nlmsg_free(msg);
11168                 return;
11169         }
11170
11171         genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink),
11172                                 msg, 0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
11173 }
11174
11175 static int
11176 devlink_trap_policer_register(struct devlink *devlink,
11177                               const struct devlink_trap_policer *policer)
11178 {
11179         struct devlink_trap_policer_item *policer_item;
11180         int err;
11181
11182         if (devlink_trap_policer_item_lookup(devlink, policer->id))
11183                 return -EEXIST;
11184
11185         policer_item = kzalloc(sizeof(*policer_item), GFP_KERNEL);
11186         if (!policer_item)
11187                 return -ENOMEM;
11188
11189         policer_item->policer = policer;
11190         policer_item->rate = policer->init_rate;
11191         policer_item->burst = policer->init_burst;
11192
11193         if (devlink->ops->trap_policer_init) {
11194                 err = devlink->ops->trap_policer_init(devlink, policer);
11195                 if (err)
11196                         goto err_policer_init;
11197         }
11198
11199         list_add_tail(&policer_item->list, &devlink->trap_policer_list);
11200         devlink_trap_policer_notify(devlink, policer_item,
11201                                     DEVLINK_CMD_TRAP_POLICER_NEW);
11202
11203         return 0;
11204
11205 err_policer_init:
11206         kfree(policer_item);
11207         return err;
11208 }
11209
11210 static void
11211 devlink_trap_policer_unregister(struct devlink *devlink,
11212                                 const struct devlink_trap_policer *policer)
11213 {
11214         struct devlink_trap_policer_item *policer_item;
11215
11216         policer_item = devlink_trap_policer_item_lookup(devlink, policer->id);
11217         if (WARN_ON_ONCE(!policer_item))
11218                 return;
11219
11220         devlink_trap_policer_notify(devlink, policer_item,
11221                                     DEVLINK_CMD_TRAP_POLICER_DEL);
11222         list_del(&policer_item->list);
11223         if (devlink->ops->trap_policer_fini)
11224                 devlink->ops->trap_policer_fini(devlink, policer);
11225         kfree(policer_item);
11226 }
11227
11228 /**
11229  * devlink_trap_policers_register - Register packet trap policers with devlink.
11230  * @devlink: devlink.
11231  * @policers: Packet trap policers.
11232  * @policers_count: Count of provided packet trap policers.
11233  *
11234  * Return: Non-zero value on failure.
11235  */
11236 int
11237 devlink_trap_policers_register(struct devlink *devlink,
11238                                const struct devlink_trap_policer *policers,
11239                                size_t policers_count)
11240 {
11241         int i, err;
11242
11243         mutex_lock(&devlink->lock);
11244         for (i = 0; i < policers_count; i++) {
11245                 const struct devlink_trap_policer *policer = &policers[i];
11246
11247                 if (WARN_ON(policer->id == 0 ||
11248                             policer->max_rate < policer->min_rate ||
11249                             policer->max_burst < policer->min_burst)) {
11250                         err = -EINVAL;
11251                         goto err_trap_policer_verify;
11252                 }
11253
11254                 err = devlink_trap_policer_register(devlink, policer);
11255                 if (err)
11256                         goto err_trap_policer_register;
11257         }
11258         mutex_unlock(&devlink->lock);
11259
11260         return 0;
11261
11262 err_trap_policer_register:
11263 err_trap_policer_verify:
11264         for (i--; i >= 0; i--)
11265                 devlink_trap_policer_unregister(devlink, &policers[i]);
11266         mutex_unlock(&devlink->lock);
11267         return err;
11268 }
11269 EXPORT_SYMBOL_GPL(devlink_trap_policers_register);
11270
11271 /**
11272  * devlink_trap_policers_unregister - Unregister packet trap policers from devlink.
11273  * @devlink: devlink.
11274  * @policers: Packet trap policers.
11275  * @policers_count: Count of provided packet trap policers.
11276  */
11277 void
11278 devlink_trap_policers_unregister(struct devlink *devlink,
11279                                  const struct devlink_trap_policer *policers,
11280                                  size_t policers_count)
11281 {
11282         int i;
11283
11284         mutex_lock(&devlink->lock);
11285         for (i = policers_count - 1; i >= 0; i--)
11286                 devlink_trap_policer_unregister(devlink, &policers[i]);
11287         mutex_unlock(&devlink->lock);
11288 }
11289 EXPORT_SYMBOL_GPL(devlink_trap_policers_unregister);
11290
11291 static void __devlink_compat_running_version(struct devlink *devlink,
11292                                              char *buf, size_t len)
11293 {
11294         const struct nlattr *nlattr;
11295         struct devlink_info_req req;
11296         struct sk_buff *msg;
11297         int rem, err;
11298
11299         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
11300         if (!msg)
11301                 return;
11302
11303         req.msg = msg;
11304         err = devlink->ops->info_get(devlink, &req, NULL);
11305         if (err)
11306                 goto free_msg;
11307
11308         nla_for_each_attr(nlattr, (void *)msg->data, msg->len, rem) {
11309                 const struct nlattr *kv;
11310                 int rem_kv;
11311
11312                 if (nla_type(nlattr) != DEVLINK_ATTR_INFO_VERSION_RUNNING)
11313                         continue;
11314
11315                 nla_for_each_nested(kv, nlattr, rem_kv) {
11316                         if (nla_type(kv) != DEVLINK_ATTR_INFO_VERSION_VALUE)
11317                                 continue;
11318
11319                         strlcat(buf, nla_data(kv), len);
11320                         strlcat(buf, " ", len);
11321                 }
11322         }
11323 free_msg:
11324         nlmsg_free(msg);
11325 }
11326
11327 static struct devlink_port *netdev_to_devlink_port(struct net_device *dev)
11328 {
11329         if (!dev->netdev_ops->ndo_get_devlink_port)
11330                 return NULL;
11331
11332         return dev->netdev_ops->ndo_get_devlink_port(dev);
11333 }
11334
11335 void devlink_compat_running_version(struct devlink *devlink,
11336                                     char *buf, size_t len)
11337 {
11338         if (!devlink->ops->info_get)
11339                 return;
11340
11341         mutex_lock(&devlink->lock);
11342         __devlink_compat_running_version(devlink, buf, len);
11343         mutex_unlock(&devlink->lock);
11344 }
11345
11346 int devlink_compat_flash_update(struct devlink *devlink, const char *file_name)
11347 {
11348         struct devlink_flash_update_params params = {};
11349         int ret;
11350
11351         if (!devlink->ops->flash_update)
11352                 return -EOPNOTSUPP;
11353
11354         ret = request_firmware(&params.fw, file_name, devlink->dev);
11355         if (ret)
11356                 return ret;
11357
11358         mutex_lock(&devlink->lock);
11359         devlink_flash_update_begin_notify(devlink);
11360         ret = devlink->ops->flash_update(devlink, &params, NULL);
11361         devlink_flash_update_end_notify(devlink);
11362         mutex_unlock(&devlink->lock);
11363
11364         release_firmware(params.fw);
11365
11366         return ret;
11367 }
11368
11369 int devlink_compat_phys_port_name_get(struct net_device *dev,
11370                                       char *name, size_t len)
11371 {
11372         struct devlink_port *devlink_port;
11373
11374         /* RTNL mutex is held here which ensures that devlink_port
11375          * instance cannot disappear in the middle. No need to take
11376          * any devlink lock as only permanent values are accessed.
11377          */
11378         ASSERT_RTNL();
11379
11380         devlink_port = netdev_to_devlink_port(dev);
11381         if (!devlink_port)
11382                 return -EOPNOTSUPP;
11383
11384         return __devlink_port_phys_port_name_get(devlink_port, name, len);
11385 }
11386
11387 int devlink_compat_switch_id_get(struct net_device *dev,
11388                                  struct netdev_phys_item_id *ppid)
11389 {
11390         struct devlink_port *devlink_port;
11391
11392         /* Caller must hold RTNL mutex or reference to dev, which ensures that
11393          * devlink_port instance cannot disappear in the middle. No need to take
11394          * any devlink lock as only permanent values are accessed.
11395          */
11396         devlink_port = netdev_to_devlink_port(dev);
11397         if (!devlink_port || !devlink_port->switch_port)
11398                 return -EOPNOTSUPP;
11399
11400         memcpy(ppid, &devlink_port->attrs.switch_id, sizeof(*ppid));
11401
11402         return 0;
11403 }
11404
11405 static void __net_exit devlink_pernet_pre_exit(struct net *net)
11406 {
11407         struct devlink *devlink;
11408         u32 actions_performed;
11409         unsigned long index;
11410         int err;
11411
11412         /* In case network namespace is getting destroyed, reload
11413          * all devlink instances from this namespace into init_net.
11414          */
11415         mutex_lock(&devlink_mutex);
11416         xa_for_each_marked(&devlinks, index, devlink, DEVLINK_REGISTERED) {
11417                 if (!devlink_try_get(devlink))
11418                         continue;
11419
11420                 if (!net_eq(devlink_net(devlink), net))
11421                         goto retry;
11422
11423                 WARN_ON(!(devlink->features & DEVLINK_F_RELOAD));
11424                 err = devlink_reload(devlink, &init_net,
11425                                      DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
11426                                      DEVLINK_RELOAD_LIMIT_UNSPEC,
11427                                      &actions_performed, NULL);
11428                 if (err && err != -EOPNOTSUPP)
11429                         pr_warn("Failed to reload devlink instance into init_net\n");
11430 retry:
11431                 devlink_put(devlink);
11432         }
11433         mutex_unlock(&devlink_mutex);
11434 }
11435
11436 static struct pernet_operations devlink_pernet_ops __net_initdata = {
11437         .pre_exit = devlink_pernet_pre_exit,
11438 };
11439
11440 static int __init devlink_init(void)
11441 {
11442         int err;
11443
11444         err = genl_register_family(&devlink_nl_family);
11445         if (err)
11446                 goto out;
11447         err = register_pernet_subsys(&devlink_pernet_ops);
11448
11449 out:
11450         WARN_ON(err);
11451         return err;
11452 }
11453
11454 subsys_initcall(devlink_init);