drivers/net/ethernet/mellanox/mlxsw/core.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/core.c
   3  * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2015 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2015 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2015 Elad Raz <eladr@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/module.h>
  39 #include <linux/device.h>
  40 #include <linux/export.h>
  41 #include <linux/err.h>
  42 #include <linux/if_link.h>
  43 #include <linux/debugfs.h>
  44 #include <linux/seq_file.h>
  45 #include <linux/u64_stats_sync.h>
  46 #include <linux/netdevice.h>
  47 #include <linux/completion.h>
  48 #include <linux/skbuff.h>
  49 #include <linux/etherdevice.h>
  50 #include <linux/types.h>
  51 #include <linux/string.h>
  52 #include <linux/gfp.h>
  53 #include <linux/random.h>
  54 #include <linux/jiffies.h>
  55 #include <linux/mutex.h>
  56 #include <linux/rcupdate.h>
  57 #include <linux/slab.h>
  58 #include <linux/workqueue.h>
  59 #include <asm/byteorder.h>
  60 #include <net/devlink.h>
  61 #include <trace/events/devlink.h>
  62
  63 #include "core.h"
  64 #include "item.h"
  65 #include "cmd.h"
  66 #include "port.h"
  67 #include "trap.h"
  68 #include "emad.h"
  69 #include "reg.h"
  70 #include "resources.h"
  71
  72 static LIST_HEAD(mlxsw_core_driver_list);
  73 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
  74
  75 static const char mlxsw_core_driver_name[] = "mlxsw_core";
  76
  77 static struct dentry *mlxsw_core_dbg_root;
  78
  79 static struct workqueue_struct *mlxsw_wq;
  80
  81 struct mlxsw_core_pcpu_stats {
  82         u64                     trap_rx_packets[MLXSW_TRAP_ID_MAX];
  83         u64                     trap_rx_bytes[MLXSW_TRAP_ID_MAX];
  84         u64                     port_rx_packets[MLXSW_PORT_MAX_PORTS];
  85         u64                     port_rx_bytes[MLXSW_PORT_MAX_PORTS];
  86         struct u64_stats_sync   syncp;
  87         u32                     trap_rx_dropped[MLXSW_TRAP_ID_MAX];
  88         u32                     port_rx_dropped[MLXSW_PORT_MAX_PORTS];
  89         u32                     trap_rx_invalid;
  90         u32                     port_rx_invalid;
  91 };
  92
  93 struct mlxsw_core_port {
  94         struct devlink_port devlink_port;
  95         void *port_driver_priv;
  96         u8 local_port;
  97 };
  98
  99 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
 100 {
 101         return mlxsw_core_port->port_driver_priv;
 102 }
 103 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
 104
 105 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
 106 {
 107         return mlxsw_core_port->port_driver_priv != NULL;
 108 }
 109
 110 struct mlxsw_core {
 111         struct mlxsw_driver *driver;
 112         const struct mlxsw_bus *bus;
 113         void *bus_priv;
 114         const struct mlxsw_bus_info *bus_info;
 115         struct list_head rx_listener_list;
 116         struct list_head event_listener_list;
 117         struct {
 118                 atomic64_t tid;
 119                 struct list_head trans_list;
 120                 spinlock_t trans_list_lock; /* protects trans_list writes */
 121                 bool use_emad;
 122         } emad;
 123         struct mlxsw_core_pcpu_stats __percpu *pcpu_stats;
 124         struct dentry *dbg_dir;
 125         struct {
 126                 struct debugfs_blob_wrapper vsd_blob;
 127                 struct debugfs_blob_wrapper psid_blob;
 128         } dbg;
 129         struct {
 130                 u8 *mapping; /* lag_id+port_index to local_port mapping */
 131         } lag;
 132         struct mlxsw_res res;
 133         struct mlxsw_hwmon *hwmon;
 134         struct mlxsw_thermal *thermal;
 135         struct mlxsw_core_port ports[MLXSW_PORT_MAX_PORTS];
 136         unsigned long driver_priv[0];
 137         /* driver_priv has to be always the last item */
 138 };
 139
 140 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 141 {
 142         return mlxsw_core->driver_priv;
 143 }
 144 EXPORT_SYMBOL(mlxsw_core_driver_priv);
 145
 146 struct mlxsw_rx_listener_item {
 147         struct list_head list;
 148         struct mlxsw_rx_listener rxl;
 149         void *priv;
 150 };
 151
 152 struct mlxsw_event_listener_item {
 153         struct list_head list;
 154         struct mlxsw_event_listener el;
 155         void *priv;
 156 };
 157
 158 /******************
 159  * EMAD processing
 160  ******************/
 161
 162 /* emad_eth_hdr_dmac
 163  * Destination MAC in EMAD's Ethernet header.
 164  * Must be set to 01:02:c9:00:00:01
 165  */
 166 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
 167
 168 /* emad_eth_hdr_smac
 169  * Source MAC in EMAD's Ethernet header.
 170  * Must be set to 00:02:c9:01:02:03
 171  */
 172 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
 173
 174 /* emad_eth_hdr_ethertype
 175  * Ethertype in EMAD's Ethernet header.
 176  * Must be set to 0x8932
 177  */
 178 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
 179
 180 /* emad_eth_hdr_mlx_proto
 181  * Mellanox protocol.
 182  * Must be set to 0x0.
 183  */
 184 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
 185
 186 /* emad_eth_hdr_ver
 187  * Mellanox protocol version.
 188  * Must be set to 0x0.
 189  */
 190 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
 191
 192 /* emad_op_tlv_type
 193  * Type of the TLV.
 194  * Must be set to 0x1 (operation TLV).
 195  */
 196 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
 197
 198 /* emad_op_tlv_len
 199  * Length of the operation TLV in u32.
 200  * Must be set to 0x4.
 201  */
 202 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
 203
 204 /* emad_op_tlv_dr
 205  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
 206  * EMAD. DR TLV must follow.
 207  *
 208  * Note: Currently not supported and must not be set.
 209  */
 210 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
 211
 212 /* emad_op_tlv_status
 213  * Returned status in case of EMAD response. Must be set to 0 in case
 214  * of EMAD request.
 215  * 0x0 - success
 216  * 0x1 - device is busy. Requester should retry
 217  * 0x2 - Mellanox protocol version not supported
 218  * 0x3 - unknown TLV
 219  * 0x4 - register not supported
 220  * 0x5 - operation class not supported
 221  * 0x6 - EMAD method not supported
 222  * 0x7 - bad parameter (e.g. port out of range)
 223  * 0x8 - resource not available
 224  * 0x9 - message receipt acknowledgment. Requester should retry
 225  * 0x70 - internal error
 226  */
 227 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
 228
 229 /* emad_op_tlv_register_id
 230  * Register ID of register within register TLV.
 231  */
 232 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
 233
 234 /* emad_op_tlv_r
 235  * Response bit. Setting to 1 indicates Response, otherwise request.
 236  */
 237 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
 238
 239 /* emad_op_tlv_method
 240  * EMAD method type.
 241  * 0x1 - query
 242  * 0x2 - write
 243  * 0x3 - send (currently not supported)
 244  * 0x4 - event
 245  */
 246 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
 247
 248 /* emad_op_tlv_class
 249  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
 250  */
 251 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
 252
 253 /* emad_op_tlv_tid
 254  * EMAD transaction ID. Used for pairing request and response EMADs.
 255  */
 256 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
 257
 258 /* emad_reg_tlv_type
 259  * Type of the TLV.
 260  * Must be set to 0x3 (register TLV).
 261  */
 262 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
 263
 264 /* emad_reg_tlv_len
 265  * Length of the operation TLV in u32.
 266  */
 267 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
 268
 269 /* emad_end_tlv_type
 270  * Type of the TLV.
 271  * Must be set to 0x0 (end TLV).
 272  */
 273 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
 274
 275 /* emad_end_tlv_len
 276  * Length of the end TLV in u32.
 277  * Must be set to 1.
 278  */
 279 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
 280
 281 enum mlxsw_core_reg_access_type {
 282         MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
 283         MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
 284 };
 285
 286 static inline const char *
 287 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
 288 {
 289         switch (type) {
 290         case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
 291                 return "query";
 292         case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
 293                 return "write";
 294         }
 295         BUG();
 296 }
 297
 298 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
 299 {
 300         mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
 301         mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
 302 }
 303
 304 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
 305                                     const struct mlxsw_reg_info *reg,
 306                                     char *payload)
 307 {
 308         mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
 309         mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
 310         memcpy(reg_tlv + sizeof(u32), payload, reg->len);
 311 }
 312
 313 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
 314                                    const struct mlxsw_reg_info *reg,
 315                                    enum mlxsw_core_reg_access_type type,
 316                                    u64 tid)
 317 {
 318         mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
 319         mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
 320         mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
 321         mlxsw_emad_op_tlv_status_set(op_tlv, 0);
 322         mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
 323         mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
 324         if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
 325                 mlxsw_emad_op_tlv_method_set(op_tlv,
 326                                              MLXSW_EMAD_OP_TLV_METHOD_QUERY);
 327         else
 328                 mlxsw_emad_op_tlv_method_set(op_tlv,
 329                                              MLXSW_EMAD_OP_TLV_METHOD_WRITE);
 330         mlxsw_emad_op_tlv_class_set(op_tlv,
 331                                     MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
 332         mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
 333 }
 334
 335 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
 336 {
 337         char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
 338
 339         mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
 340         mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
 341         mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
 342         mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
 343         mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
 344
 345         skb_reset_mac_header(skb);
 346
 347         return 0;
 348 }
 349
 350 static void mlxsw_emad_construct(struct sk_buff *skb,
 351                                  const struct mlxsw_reg_info *reg,
 352                                  char *payload,
 353                                  enum mlxsw_core_reg_access_type type,
 354                                  u64 tid)
 355 {
 356         char *buf;
 357
 358         buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
 359         mlxsw_emad_pack_end_tlv(buf);
 360
 361         buf = skb_push(skb, reg->len + sizeof(u32));
 362         mlxsw_emad_pack_reg_tlv(buf, reg, payload);
 363
 364         buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
 365         mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
 366
 367         mlxsw_emad_construct_eth_hdr(skb);
 368 }
 369
 370 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
 371 {
 372         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN));
 373 }
 374
 375 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
 376 {
 377         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN +
 378                                       MLXSW_EMAD_OP_TLV_LEN * sizeof(u32)));
 379 }
 380
 381 static char *mlxsw_emad_reg_payload(const char *op_tlv)
 382 {
 383         return ((char *) (op_tlv + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
 384 }
 385
 386 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
 387 {
 388         char *op_tlv;
 389
 390         op_tlv = mlxsw_emad_op_tlv(skb);
 391         return mlxsw_emad_op_tlv_tid_get(op_tlv);
 392 }
 393
 394 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
 395 {
 396         char *op_tlv;
 397
 398         op_tlv = mlxsw_emad_op_tlv(skb);
 399         return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
 400 }
 401
 402 static int mlxsw_emad_process_status(char *op_tlv,
 403                                      enum mlxsw_emad_op_tlv_status *p_status)
 404 {
 405         *p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
 406
 407         switch (*p_status) {
 408         case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
 409                 return 0;
 410         case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
 411         case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
 412                 return -EAGAIN;
 413         case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
 414         case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
 415         case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
 416         case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
 417         case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
 418         case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
 419         case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
 420         case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
 421         default:
 422                 return -EIO;
 423         }
 424 }
 425
 426 static int
 427 mlxsw_emad_process_status_skb(struct sk_buff *skb,
 428                               enum mlxsw_emad_op_tlv_status *p_status)
 429 {
 430         return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
 431 }
 432
 433 struct mlxsw_reg_trans {
 434         struct list_head list;
 435         struct list_head bulk_list;
 436         struct mlxsw_core *core;
 437         struct sk_buff *tx_skb;
 438         struct mlxsw_tx_info tx_info;
 439         struct delayed_work timeout_dw;
 440         unsigned int retries;
 441         u64 tid;
 442         struct completion completion;
 443         atomic_t active;
 444         mlxsw_reg_trans_cb_t *cb;
 445         unsigned long cb_priv;
 446         const struct mlxsw_reg_info *reg;
 447         enum mlxsw_core_reg_access_type type;
 448         int err;
 449         enum mlxsw_emad_op_tlv_status emad_status;
 450         struct rcu_head rcu;
 451 };
 452
 453 #define MLXSW_EMAD_TIMEOUT_MS 200
 454
 455 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
 456 {
 457         unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
 458
 459         mlxsw_core_schedule_dw(&trans->timeout_dw, timeout);
 460 }
 461
 462 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
 463                                struct mlxsw_reg_trans *trans)
 464 {
 465         struct sk_buff *skb;
 466         int err;
 467
 468         skb = skb_copy(trans->tx_skb, GFP_KERNEL);
 469         if (!skb)
 470                 return -ENOMEM;
 471
 472         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
 473                             skb->data + mlxsw_core->driver->txhdr_len,
 474                             skb->len - mlxsw_core->driver->txhdr_len);
 475
 476         atomic_set(&trans->active, 1);
 477         err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
 478         if (err) {
 479                 dev_kfree_skb(skb);
 480                 return err;
 481         }
 482         mlxsw_emad_trans_timeout_schedule(trans);
 483         return 0;
 484 }
 485
 486 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
 487 {
 488         struct mlxsw_core *mlxsw_core = trans->core;
 489
 490         dev_kfree_skb(trans->tx_skb);
 491         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 492         list_del_rcu(&trans->list);
 493         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 494         trans->err = err;
 495         complete(&trans->completion);
 496 }
 497
 498 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
 499                                       struct mlxsw_reg_trans *trans)
 500 {
 501         int err;
 502
 503         if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
 504                 trans->retries++;
 505                 err = mlxsw_emad_transmit(trans->core, trans);
 506                 if (err == 0)
 507                         return;
 508         } else {
 509                 err = -EIO;
 510         }
 511         mlxsw_emad_trans_finish(trans, err);
 512 }
 513
 514 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
 515 {
 516         struct mlxsw_reg_trans *trans = container_of(work,
 517                                                      struct mlxsw_reg_trans,
 518                                                      timeout_dw.work);
 519
 520         if (!atomic_dec_and_test(&trans->active))
 521                 return;
 522
 523         mlxsw_emad_transmit_retry(trans->core, trans);
 524 }
 525
 526 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
 527                                         struct mlxsw_reg_trans *trans,
 528                                         struct sk_buff *skb)
 529 {
 530         int err;
 531
 532         if (!atomic_dec_and_test(&trans->active))
 533                 return;
 534
 535         err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
 536         if (err == -EAGAIN) {
 537                 mlxsw_emad_transmit_retry(mlxsw_core, trans);
 538         } else {
 539                 if (err == 0) {
 540                         char *op_tlv = mlxsw_emad_op_tlv(skb);
 541
 542                         if (trans->cb)
 543                                 trans->cb(mlxsw_core,
 544                                           mlxsw_emad_reg_payload(op_tlv),
 545                                           trans->reg->len, trans->cb_priv);
 546                 }
 547                 mlxsw_emad_trans_finish(trans, err);
 548         }
 549 }
 550
 551 /* called with rcu read lock held */
 552 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port,
 553                                         void *priv)
 554 {
 555         struct mlxsw_core *mlxsw_core = priv;
 556         struct mlxsw_reg_trans *trans;
 557
 558         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
 559                             skb->data, skb->len);
 560
 561         if (!mlxsw_emad_is_resp(skb))
 562                 goto free_skb;
 563
 564         list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
 565                 if (mlxsw_emad_get_tid(skb) == trans->tid) {
 566                         mlxsw_emad_process_response(mlxsw_core, trans, skb);
 567                         break;
 568                 }
 569         }
 570
 571 free_skb:
 572         dev_kfree_skb(skb);
 573 }
 574
 575 static const struct mlxsw_listener mlxsw_emad_rx_listener =
 576         MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
 577                   EMAD, DISCARD);
 578
 579 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
 580 {
 581         u64 tid;
 582         int err;
 583
 584         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 585                 return 0;
 586
 587         /* Set the upper 32 bits of the transaction ID field to a random
 588          * number. This allows us to discard EMADs addressed to other
 589          * devices.
 590          */
 591         get_random_bytes(&tid, 4);
 592         tid <<= 32;
 593         atomic64_set(&mlxsw_core->emad.tid, tid);
 594
 595         INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
 596         spin_lock_init(&mlxsw_core->emad.trans_list_lock);
 597
 598         err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
 599                                        mlxsw_core);
 600         if (err)
 601                 return err;
 602
 603         err = mlxsw_core->driver->basic_trap_groups_set(mlxsw_core);
 604         if (err)
 605                 goto err_emad_trap_set;
 606         mlxsw_core->emad.use_emad = true;
 607
 608         return 0;
 609
 610 err_emad_trap_set:
 611         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 612                                    mlxsw_core);
 613         return err;
 614 }
 615
 616 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
 617 {
 618
 619         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 620                 return;
 621
 622         mlxsw_core->emad.use_emad = false;
 623         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 624                                    mlxsw_core);
 625 }
 626
 627 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
 628                                         u16 reg_len)
 629 {
 630         struct sk_buff *skb;
 631         u16 emad_len;
 632
 633         emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
 634                     (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
 635                     sizeof(u32) + mlxsw_core->driver->txhdr_len);
 636         if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
 637                 return NULL;
 638
 639         skb = netdev_alloc_skb(NULL, emad_len);
 640         if (!skb)
 641                 return NULL;
 642         memset(skb->data, 0, emad_len);
 643         skb_reserve(skb, emad_len);
 644
 645         return skb;
 646 }
 647
 648 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
 649                                  const struct mlxsw_reg_info *reg,
 650                                  char *payload,
 651                                  enum mlxsw_core_reg_access_type type,
 652                                  struct mlxsw_reg_trans *trans,
 653                                  struct list_head *bulk_list,
 654                                  mlxsw_reg_trans_cb_t *cb,
 655                                  unsigned long cb_priv, u64 tid)
 656 {
 657         struct sk_buff *skb;
 658         int err;
 659
 660         dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
 661                 trans->tid, reg->id, mlxsw_reg_id_str(reg->id),
 662                 mlxsw_core_reg_access_type_str(type));
 663
 664         skb = mlxsw_emad_alloc(mlxsw_core, reg->len);
 665         if (!skb)
 666                 return -ENOMEM;
 667
 668         list_add_tail(&trans->bulk_list, bulk_list);
 669         trans->core = mlxsw_core;
 670         trans->tx_skb = skb;
 671         trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
 672         trans->tx_info.is_emad = true;
 673         INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
 674         trans->tid = tid;
 675         init_completion(&trans->completion);
 676         trans->cb = cb;
 677         trans->cb_priv = cb_priv;
 678         trans->reg = reg;
 679         trans->type = type;
 680
 681         mlxsw_emad_construct(skb, reg, payload, type, trans->tid);
 682         mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
 683
 684         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 685         list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
 686         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 687         err = mlxsw_emad_transmit(mlxsw_core, trans);
 688         if (err)
 689                 goto err_out;
 690         return 0;
 691
 692 err_out:
 693         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 694         list_del_rcu(&trans->list);
 695         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 696         list_del(&trans->bulk_list);
 697         dev_kfree_skb(trans->tx_skb);
 698         return err;
 699 }
 700
 701 /*****************
 702  * Core functions
 703  *****************/
 704
 705 static int mlxsw_core_rx_stats_dbg_read(struct seq_file *file, void *data)
 706 {
 707         struct mlxsw_core *mlxsw_core = file->private;
 708         struct mlxsw_core_pcpu_stats *p;
 709         u64 rx_packets, rx_bytes;
 710         u64 tmp_rx_packets, tmp_rx_bytes;
 711         u32 rx_dropped, rx_invalid;
 712         unsigned int start;
 713         int i;
 714         int j;
 715         static const char hdr[] =
 716                 "     NUM   RX_PACKETS     RX_BYTES RX_DROPPED\n";
 717
 718         seq_printf(file, hdr);
 719         for (i = 0; i < MLXSW_TRAP_ID_MAX; i++) {
 720                 rx_packets = 0;
 721                 rx_bytes = 0;
 722                 rx_dropped = 0;
 723                 for_each_possible_cpu(j) {
 724                         p = per_cpu_ptr(mlxsw_core->pcpu_stats, j);
 725                         do {
 726                                 start = u64_stats_fetch_begin(&p->syncp);
 727                                 tmp_rx_packets = p->trap_rx_packets[i];
 728                                 tmp_rx_bytes = p->trap_rx_bytes[i];
 729                         } while (u64_stats_fetch_retry(&p->syncp, start));
 730
 731                         rx_packets += tmp_rx_packets;
 732                         rx_bytes += tmp_rx_bytes;
 733                         rx_dropped += p->trap_rx_dropped[i];
 734                 }
 735                 seq_printf(file, "trap %3d %12llu %12llu %10u\n",
 736                            i, rx_packets, rx_bytes, rx_dropped);
 737         }
 738         rx_invalid = 0;
 739         for_each_possible_cpu(j) {
 740                 p = per_cpu_ptr(mlxsw_core->pcpu_stats, j);
 741                 rx_invalid += p->trap_rx_invalid;
 742         }
 743         seq_printf(file, "trap INV                           %10u\n",
 744                    rx_invalid);
 745
 746         for (i = 0; i < MLXSW_PORT_MAX_PORTS; i++) {
 747                 rx_packets = 0;
 748                 rx_bytes = 0;
 749                 rx_dropped = 0;
 750                 for_each_possible_cpu(j) {
 751                         p = per_cpu_ptr(mlxsw_core->pcpu_stats, j);
 752                         do {
 753                                 start = u64_stats_fetch_begin(&p->syncp);
 754                                 tmp_rx_packets = p->port_rx_packets[i];
 755                                 tmp_rx_bytes = p->port_rx_bytes[i];
 756                         } while (u64_stats_fetch_retry(&p->syncp, start));
 757
 758                         rx_packets += tmp_rx_packets;
 759                         rx_bytes += tmp_rx_bytes;
 760                         rx_dropped += p->port_rx_dropped[i];
 761                 }
 762                 seq_printf(file, "port %3d %12llu %12llu %10u\n",
 763                            i, rx_packets, rx_bytes, rx_dropped);
 764         }
 765         rx_invalid = 0;
 766         for_each_possible_cpu(j) {
 767                 p = per_cpu_ptr(mlxsw_core->pcpu_stats, j);
 768                 rx_invalid += p->port_rx_invalid;
 769         }
 770         seq_printf(file, "port INV                           %10u\n",
 771                    rx_invalid);
 772         return 0;
 773 }
 774
 775 static int mlxsw_core_rx_stats_dbg_open(struct inode *inode, struct file *f)
 776 {
 777         struct mlxsw_core *mlxsw_core = inode->i_private;
 778
 779         return single_open(f, mlxsw_core_rx_stats_dbg_read, mlxsw_core);
 780 }
 781
 782 static const struct file_operations mlxsw_core_rx_stats_dbg_ops = {
 783         .owner = THIS_MODULE,
 784         .open = mlxsw_core_rx_stats_dbg_open,
 785         .release = single_release,
 786         .read = seq_read,
 787         .llseek = seq_lseek
 788 };
 789
 790 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
 791 {
 792         spin_lock(&mlxsw_core_driver_list_lock);
 793         list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
 794         spin_unlock(&mlxsw_core_driver_list_lock);
 795         return 0;
 796 }
 797 EXPORT_SYMBOL(mlxsw_core_driver_register);
 798
 799 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
 800 {
 801         spin_lock(&mlxsw_core_driver_list_lock);
 802         list_del(&mlxsw_driver->list);
 803         spin_unlock(&mlxsw_core_driver_list_lock);
 804 }
 805 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
 806
 807 static struct mlxsw_driver *__driver_find(const char *kind)
 808 {
 809         struct mlxsw_driver *mlxsw_driver;
 810
 811         list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
 812                 if (strcmp(mlxsw_driver->kind, kind) == 0)
 813                         return mlxsw_driver;
 814         }
 815         return NULL;
 816 }
 817
 818 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
 819 {
 820         struct mlxsw_driver *mlxsw_driver;
 821
 822         spin_lock(&mlxsw_core_driver_list_lock);
 823         mlxsw_driver = __driver_find(kind);
 824         spin_unlock(&mlxsw_core_driver_list_lock);
 825         return mlxsw_driver;
 826 }
 827
 828 static void mlxsw_core_driver_put(const char *kind)
 829 {
 830         struct mlxsw_driver *mlxsw_driver;
 831
 832         spin_lock(&mlxsw_core_driver_list_lock);
 833         mlxsw_driver = __driver_find(kind);
 834         spin_unlock(&mlxsw_core_driver_list_lock);
 835 }
 836
 837 static int mlxsw_core_debugfs_init(struct mlxsw_core *mlxsw_core)
 838 {
 839         const struct mlxsw_bus_info *bus_info = mlxsw_core->bus_info;
 840
 841         mlxsw_core->dbg_dir = debugfs_create_dir(bus_info->device_name,
 842                                                  mlxsw_core_dbg_root);
 843         if (!mlxsw_core->dbg_dir)
 844                 return -ENOMEM;
 845         debugfs_create_file("rx_stats", S_IRUGO, mlxsw_core->dbg_dir,
 846                             mlxsw_core, &mlxsw_core_rx_stats_dbg_ops);
 847         mlxsw_core->dbg.vsd_blob.data = (void *) &bus_info->vsd;
 848         mlxsw_core->dbg.vsd_blob.size = sizeof(bus_info->vsd);
 849         debugfs_create_blob("vsd", S_IRUGO, mlxsw_core->dbg_dir,
 850                             &mlxsw_core->dbg.vsd_blob);
 851         mlxsw_core->dbg.psid_blob.data = (void *) &bus_info->psid;
 852         mlxsw_core->dbg.psid_blob.size = sizeof(bus_info->psid);
 853         debugfs_create_blob("psid", S_IRUGO, mlxsw_core->dbg_dir,
 854                             &mlxsw_core->dbg.psid_blob);
 855         return 0;
 856 }
 857
 858 static void mlxsw_core_debugfs_fini(struct mlxsw_core *mlxsw_core)
 859 {
 860         debugfs_remove_recursive(mlxsw_core->dbg_dir);
 861 }
 862
 863 static int mlxsw_devlink_port_split(struct devlink *devlink,
 864                                     unsigned int port_index,
 865                                     unsigned int count)
 866 {
 867         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 868
 869         if (port_index >= MLXSW_PORT_MAX_PORTS)
 870                 return -EINVAL;
 871         if (!mlxsw_core->driver->port_split)
 872                 return -EOPNOTSUPP;
 873         return mlxsw_core->driver->port_split(mlxsw_core, port_index, count);
 874 }
 875
 876 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
 877                                       unsigned int port_index)
 878 {
 879         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 880
 881         if (port_index >= MLXSW_PORT_MAX_PORTS)
 882                 return -EINVAL;
 883         if (!mlxsw_core->driver->port_unsplit)
 884                 return -EOPNOTSUPP;
 885         return mlxsw_core->driver->port_unsplit(mlxsw_core, port_index);
 886 }
 887
 888 static int
 889 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
 890                           unsigned int sb_index, u16 pool_index,
 891                           struct devlink_sb_pool_info *pool_info)
 892 {
 893         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 894         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 895
 896         if (!mlxsw_driver->sb_pool_get)
 897                 return -EOPNOTSUPP;
 898         return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
 899                                          pool_index, pool_info);
 900 }
 901
 902 static int
 903 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
 904                           unsigned int sb_index, u16 pool_index, u32 size,
 905                           enum devlink_sb_threshold_type threshold_type)
 906 {
 907         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 908         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 909
 910         if (!mlxsw_driver->sb_pool_set)
 911                 return -EOPNOTSUPP;
 912         return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
 913                                          pool_index, size, threshold_type);
 914 }
 915
 916 static void *__dl_port(struct devlink_port *devlink_port)
 917 {
 918         return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
 919 }
 920
 921 static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
 922                                        enum devlink_port_type port_type)
 923 {
 924         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 925         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 926         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 927
 928         if (!mlxsw_driver->port_type_set)
 929                 return -EOPNOTSUPP;
 930
 931         return mlxsw_driver->port_type_set(mlxsw_core,
 932                                            mlxsw_core_port->local_port,
 933                                            port_type);
 934 }
 935
 936 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
 937                                           unsigned int sb_index, u16 pool_index,
 938                                           u32 *p_threshold)
 939 {
 940         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 941         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 942         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 943
 944         if (!mlxsw_driver->sb_port_pool_get ||
 945             !mlxsw_core_port_check(mlxsw_core_port))
 946                 return -EOPNOTSUPP;
 947         return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
 948                                               pool_index, p_threshold);
 949 }
 950
 951 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
 952                                           unsigned int sb_index, u16 pool_index,
 953                                           u32 threshold)
 954 {
 955         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 956         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 957         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 958
 959         if (!mlxsw_driver->sb_port_pool_set ||
 960             !mlxsw_core_port_check(mlxsw_core_port))
 961                 return -EOPNOTSUPP;
 962         return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
 963                                               pool_index, threshold);
 964 }
 965
 966 static int
 967 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
 968                                   unsigned int sb_index, u16 tc_index,
 969                                   enum devlink_sb_pool_type pool_type,
 970                                   u16 *p_pool_index, u32 *p_threshold)
 971 {
 972         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 973         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 974         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 975
 976         if (!mlxsw_driver->sb_tc_pool_bind_get ||
 977             !mlxsw_core_port_check(mlxsw_core_port))
 978                 return -EOPNOTSUPP;
 979         return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
 980                                                  tc_index, pool_type,
 981                                                  p_pool_index, p_threshold);
 982 }
 983
 984 static int
 985 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
 986                                   unsigned int sb_index, u16 tc_index,
 987                                   enum devlink_sb_pool_type pool_type,
 988                                   u16 pool_index, u32 threshold)
 989 {
 990         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 991         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 992         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 993
 994         if (!mlxsw_driver->sb_tc_pool_bind_set ||
 995             !mlxsw_core_port_check(mlxsw_core_port))
 996                 return -EOPNOTSUPP;
 997         return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
 998                                                  tc_index, pool_type,
 999                                                  pool_index, threshold);
1000 }
1001
1002 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
1003                                          unsigned int sb_index)
1004 {
1005         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1006         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1007
1008         if (!mlxsw_driver->sb_occ_snapshot)
1009                 return -EOPNOTSUPP;
1010         return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
1011 }
1012
1013 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
1014                                           unsigned int sb_index)
1015 {
1016         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1017         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1018
1019         if (!mlxsw_driver->sb_occ_max_clear)
1020                 return -EOPNOTSUPP;
1021         return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
1022 }
1023
1024 static int
1025 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
1026                                    unsigned int sb_index, u16 pool_index,
1027                                    u32 *p_cur, u32 *p_max)
1028 {
1029         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1030         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1031         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1032
1033         if (!mlxsw_driver->sb_occ_port_pool_get ||
1034             !mlxsw_core_port_check(mlxsw_core_port))
1035                 return -EOPNOTSUPP;
1036         return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
1037                                                   pool_index, p_cur, p_max);
1038 }
1039
1040 static int
1041 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
1042                                       unsigned int sb_index, u16 tc_index,
1043                                       enum devlink_sb_pool_type pool_type,
1044                                       u32 *p_cur, u32 *p_max)
1045 {
1046         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
1047         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1048         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
1049
1050         if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
1051             !mlxsw_core_port_check(mlxsw_core_port))
1052                 return -EOPNOTSUPP;
1053         return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
1054                                                      sb_index, tc_index,
1055                                                      pool_type, p_cur, p_max);
1056 }
1057
1058 static const struct devlink_ops mlxsw_devlink_ops = {
1059         .port_type_set                  = mlxsw_devlink_port_type_set,
1060         .port_split                     = mlxsw_devlink_port_split,
1061         .port_unsplit                   = mlxsw_devlink_port_unsplit,
1062         .sb_pool_get                    = mlxsw_devlink_sb_pool_get,
1063         .sb_pool_set                    = mlxsw_devlink_sb_pool_set,
1064         .sb_port_pool_get               = mlxsw_devlink_sb_port_pool_get,
1065         .sb_port_pool_set               = mlxsw_devlink_sb_port_pool_set,
1066         .sb_tc_pool_bind_get            = mlxsw_devlink_sb_tc_pool_bind_get,
1067         .sb_tc_pool_bind_set            = mlxsw_devlink_sb_tc_pool_bind_set,
1068         .sb_occ_snapshot                = mlxsw_devlink_sb_occ_snapshot,
1069         .sb_occ_max_clear               = mlxsw_devlink_sb_occ_max_clear,
1070         .sb_occ_port_pool_get           = mlxsw_devlink_sb_occ_port_pool_get,
1071         .sb_occ_tc_port_bind_get        = mlxsw_devlink_sb_occ_tc_port_bind_get,
1072 };
1073
1074 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1075                                    const struct mlxsw_bus *mlxsw_bus,
1076                                    void *bus_priv)
1077 {
1078         const char *device_kind = mlxsw_bus_info->device_kind;
1079         struct mlxsw_core *mlxsw_core;
1080         struct mlxsw_driver *mlxsw_driver;
1081         struct devlink *devlink;
1082         size_t alloc_size;
1083         int err;
1084
1085         mlxsw_driver = mlxsw_core_driver_get(device_kind);
1086         if (!mlxsw_driver)
1087                 return -EINVAL;
1088         alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
1089         devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
1090         if (!devlink) {
1091                 err = -ENOMEM;
1092                 goto err_devlink_alloc;
1093         }
1094
1095         mlxsw_core = devlink_priv(devlink);
1096         INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
1097         INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
1098         mlxsw_core->driver = mlxsw_driver;
1099         mlxsw_core->bus = mlxsw_bus;
1100         mlxsw_core->bus_priv = bus_priv;
1101         mlxsw_core->bus_info = mlxsw_bus_info;
1102
1103         mlxsw_core->pcpu_stats =
1104                 netdev_alloc_pcpu_stats(struct mlxsw_core_pcpu_stats);
1105         if (!mlxsw_core->pcpu_stats) {
1106                 err = -ENOMEM;
1107                 goto err_alloc_stats;
1108         }
1109
1110         err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile,
1111                               &mlxsw_core->res);
1112         if (err)
1113                 goto err_bus_init;
1114
1115         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
1116             MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
1117                 alloc_size = sizeof(u8) *
1118                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
1119                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
1120                 mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
1121                 if (!mlxsw_core->lag.mapping) {
1122                         err = -ENOMEM;
1123                         goto err_alloc_lag_mapping;
1124                 }
1125         }
1126
1127         err = mlxsw_emad_init(mlxsw_core);
1128         if (err)
1129                 goto err_emad_init;
1130
1131         err = devlink_register(devlink, mlxsw_bus_info->dev);
1132         if (err)
1133                 goto err_devlink_register;
1134
1135         err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
1136         if (err)
1137                 goto err_hwmon_init;
1138
1139         err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
1140                                  &mlxsw_core->thermal);
1141         if (err)
1142                 goto err_thermal_init;
1143
1144         if (mlxsw_driver->init) {
1145                 err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info);
1146                 if (err)
1147                         goto err_driver_init;
1148         }
1149
1150         err = mlxsw_core_debugfs_init(mlxsw_core);
1151         if (err)
1152                 goto err_debugfs_init;
1153
1154         return 0;
1155
1156 err_debugfs_init:
1157         if (mlxsw_core->driver->fini)
1158                 mlxsw_core->driver->fini(mlxsw_core);
1159 err_driver_init:
1160         mlxsw_thermal_fini(mlxsw_core->thermal);
1161 err_thermal_init:
1162 err_hwmon_init:
1163         devlink_unregister(devlink);
1164 err_devlink_register:
1165         mlxsw_emad_fini(mlxsw_core);
1166 err_emad_init:
1167         kfree(mlxsw_core->lag.mapping);
1168 err_alloc_lag_mapping:
1169         mlxsw_bus->fini(bus_priv);
1170 err_bus_init:
1171         free_percpu(mlxsw_core->pcpu_stats);
1172 err_alloc_stats:
1173         devlink_free(devlink);
1174 err_devlink_alloc:
1175         mlxsw_core_driver_put(device_kind);
1176         return err;
1177 }
1178 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
1179
1180 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core)
1181 {
1182         const char *device_kind = mlxsw_core->bus_info->device_kind;
1183         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1184
1185         mlxsw_core_debugfs_fini(mlxsw_core);
1186         if (mlxsw_core->driver->fini)
1187                 mlxsw_core->driver->fini(mlxsw_core);
1188         mlxsw_thermal_fini(mlxsw_core->thermal);
1189         devlink_unregister(devlink);
1190         mlxsw_emad_fini(mlxsw_core);
1191         mlxsw_core->bus->fini(mlxsw_core->bus_priv);
1192         kfree(mlxsw_core->lag.mapping);
1193         free_percpu(mlxsw_core->pcpu_stats);
1194         devlink_free(devlink);
1195         mlxsw_core_driver_put(device_kind);
1196 }
1197 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
1198
1199 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
1200                                   const struct mlxsw_tx_info *tx_info)
1201 {
1202         return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
1203                                                   tx_info);
1204 }
1205 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
1206
1207 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1208                             const struct mlxsw_tx_info *tx_info)
1209 {
1210         return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
1211                                              tx_info);
1212 }
1213 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
1214
1215 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
1216                                    const struct mlxsw_rx_listener *rxl_b)
1217 {
1218         return (rxl_a->func == rxl_b->func &&
1219                 rxl_a->local_port == rxl_b->local_port &&
1220                 rxl_a->trap_id == rxl_b->trap_id);
1221 }
1222
1223 static struct mlxsw_rx_listener_item *
1224 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
1225                         const struct mlxsw_rx_listener *rxl,
1226                         void *priv)
1227 {
1228         struct mlxsw_rx_listener_item *rxl_item;
1229
1230         list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
1231                 if (__is_rx_listener_equal(&rxl_item->rxl, rxl) &&
1232                     rxl_item->priv == priv)
1233                         return rxl_item;
1234         }
1235         return NULL;
1236 }
1237
1238 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
1239                                     const struct mlxsw_rx_listener *rxl,
1240                                     void *priv)
1241 {
1242         struct mlxsw_rx_listener_item *rxl_item;
1243
1244         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1245         if (rxl_item)
1246                 return -EEXIST;
1247         rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
1248         if (!rxl_item)
1249                 return -ENOMEM;
1250         rxl_item->rxl = *rxl;
1251         rxl_item->priv = priv;
1252
1253         list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
1254         return 0;
1255 }
1256 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
1257
1258 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
1259                                        const struct mlxsw_rx_listener *rxl,
1260                                        void *priv)
1261 {
1262         struct mlxsw_rx_listener_item *rxl_item;
1263
1264         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1265         if (!rxl_item)
1266                 return;
1267         list_del_rcu(&rxl_item->list);
1268         synchronize_rcu();
1269         kfree(rxl_item);
1270 }
1271 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
1272
1273 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port,
1274                                            void *priv)
1275 {
1276         struct mlxsw_event_listener_item *event_listener_item = priv;
1277         struct mlxsw_reg_info reg;
1278         char *payload;
1279         char *op_tlv = mlxsw_emad_op_tlv(skb);
1280         char *reg_tlv = mlxsw_emad_reg_tlv(skb);
1281
1282         reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
1283         reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
1284         payload = mlxsw_emad_reg_payload(op_tlv);
1285         event_listener_item->el.func(&reg, payload, event_listener_item->priv);
1286         dev_kfree_skb(skb);
1287 }
1288
1289 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
1290                                       const struct mlxsw_event_listener *el_b)
1291 {
1292         return (el_a->func == el_b->func &&
1293                 el_a->trap_id == el_b->trap_id);
1294 }
1295
1296 static struct mlxsw_event_listener_item *
1297 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
1298                            const struct mlxsw_event_listener *el,
1299                            void *priv)
1300 {
1301         struct mlxsw_event_listener_item *el_item;
1302
1303         list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
1304                 if (__is_event_listener_equal(&el_item->el, el) &&
1305                     el_item->priv == priv)
1306                         return el_item;
1307         }
1308         return NULL;
1309 }
1310
1311 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
1312                                        const struct mlxsw_event_listener *el,
1313                                        void *priv)
1314 {
1315         int err;
1316         struct mlxsw_event_listener_item *el_item;
1317         const struct mlxsw_rx_listener rxl = {
1318                 .func = mlxsw_core_event_listener_func,
1319                 .local_port = MLXSW_PORT_DONT_CARE,
1320                 .trap_id = el->trap_id,
1321         };
1322
1323         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1324         if (el_item)
1325                 return -EEXIST;
1326         el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
1327         if (!el_item)
1328                 return -ENOMEM;
1329         el_item->el = *el;
1330         el_item->priv = priv;
1331
1332         err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item);
1333         if (err)
1334                 goto err_rx_listener_register;
1335
1336         /* No reason to save item if we did not manage to register an RX
1337          * listener for it.
1338          */
1339         list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
1340
1341         return 0;
1342
1343 err_rx_listener_register:
1344         kfree(el_item);
1345         return err;
1346 }
1347 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
1348
1349 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
1350                                           const struct mlxsw_event_listener *el,
1351                                           void *priv)
1352 {
1353         struct mlxsw_event_listener_item *el_item;
1354         const struct mlxsw_rx_listener rxl = {
1355                 .func = mlxsw_core_event_listener_func,
1356                 .local_port = MLXSW_PORT_DONT_CARE,
1357                 .trap_id = el->trap_id,
1358         };
1359
1360         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1361         if (!el_item)
1362                 return;
1363         mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl, el_item);
1364         list_del(&el_item->list);
1365         kfree(el_item);
1366 }
1367 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
1368
1369 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
1370                                         const struct mlxsw_listener *listener,
1371                                         void *priv)
1372 {
1373         if (listener->is_event)
1374                 return mlxsw_core_event_listener_register(mlxsw_core,
1375                                                 &listener->u.event_listener,
1376                                                 priv);
1377         else
1378                 return mlxsw_core_rx_listener_register(mlxsw_core,
1379                                                 &listener->u.rx_listener,
1380                                                 priv);
1381 }
1382
1383 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
1384                                       const struct mlxsw_listener *listener,
1385                                       void *priv)
1386 {
1387         if (listener->is_event)
1388                 mlxsw_core_event_listener_unregister(mlxsw_core,
1389                                                      &listener->u.event_listener,
1390                                                      priv);
1391         else
1392                 mlxsw_core_rx_listener_unregister(mlxsw_core,
1393                                                   &listener->u.rx_listener,
1394                                                   priv);
1395 }
1396
1397 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
1398                              const struct mlxsw_listener *listener, void *priv)
1399 {
1400         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1401         int err;
1402
1403         err = mlxsw_core_listener_register(mlxsw_core, listener, priv);
1404         if (err)
1405                 return err;
1406
1407         mlxsw_reg_hpkt_pack(hpkt_pl, listener->action, listener->trap_id,
1408                             listener->trap_group, listener->is_ctrl);
1409         err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
1410         if (err)
1411                 goto err_trap_set;
1412
1413         return 0;
1414
1415 err_trap_set:
1416         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1417         return err;
1418 }
1419 EXPORT_SYMBOL(mlxsw_core_trap_register);
1420
1421 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
1422                                 const struct mlxsw_listener *listener,
1423                                 void *priv)
1424 {
1425         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1426
1427         if (!listener->is_event) {
1428                 mlxsw_reg_hpkt_pack(hpkt_pl, listener->unreg_action,
1429                                     listener->trap_id, listener->trap_group,
1430                                     listener->is_ctrl);
1431                 mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1432         }
1433
1434         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1435 }
1436 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
1437
1438 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
1439 {
1440         return atomic64_inc_return(&mlxsw_core->emad.tid);
1441 }
1442
1443 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
1444                                       const struct mlxsw_reg_info *reg,
1445                                       char *payload,
1446                                       enum mlxsw_core_reg_access_type type,
1447                                       struct list_head *bulk_list,
1448                                       mlxsw_reg_trans_cb_t *cb,
1449                                       unsigned long cb_priv)
1450 {
1451         u64 tid = mlxsw_core_tid_get(mlxsw_core);
1452         struct mlxsw_reg_trans *trans;
1453         int err;
1454
1455         trans = kzalloc(sizeof(*trans), GFP_KERNEL);
1456         if (!trans)
1457                 return -ENOMEM;
1458
1459         err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
1460                                     bulk_list, cb, cb_priv, tid);
1461         if (err) {
1462                 kfree(trans);
1463                 return err;
1464         }
1465         return 0;
1466 }
1467
1468 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
1469                           const struct mlxsw_reg_info *reg, char *payload,
1470                           struct list_head *bulk_list,
1471                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1472 {
1473         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1474                                           MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
1475                                           bulk_list, cb, cb_priv);
1476 }
1477 EXPORT_SYMBOL(mlxsw_reg_trans_query);
1478
1479 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
1480                           const struct mlxsw_reg_info *reg, char *payload,
1481                           struct list_head *bulk_list,
1482                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1483 {
1484         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1485                                           MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
1486                                           bulk_list, cb, cb_priv);
1487 }
1488 EXPORT_SYMBOL(mlxsw_reg_trans_write);
1489
1490 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
1491 {
1492         struct mlxsw_core *mlxsw_core = trans->core;
1493         int err;
1494
1495         wait_for_completion(&trans->completion);
1496         cancel_delayed_work_sync(&trans->timeout_dw);
1497         err = trans->err;
1498
1499         if (trans->retries)
1500                 dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
1501                          trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
1502         if (err)
1503                 dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
1504                         trans->tid, trans->reg->id,
1505                         mlxsw_reg_id_str(trans->reg->id),
1506                         mlxsw_core_reg_access_type_str(trans->type),
1507                         trans->emad_status,
1508                         mlxsw_emad_op_tlv_status_str(trans->emad_status));
1509
1510         list_del(&trans->bulk_list);
1511         kfree_rcu(trans, rcu);
1512         return err;
1513 }
1514
1515 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
1516 {
1517         struct mlxsw_reg_trans *trans;
1518         struct mlxsw_reg_trans *tmp;
1519         int sum_err = 0;
1520         int err;
1521
1522         list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
1523                 err = mlxsw_reg_trans_wait(trans);
1524                 if (err && sum_err == 0)
1525                         sum_err = err; /* first error to be returned */
1526         }
1527         return sum_err;
1528 }
1529 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
1530
1531 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
1532                                      const struct mlxsw_reg_info *reg,
1533                                      char *payload,
1534                                      enum mlxsw_core_reg_access_type type)
1535 {
1536         enum mlxsw_emad_op_tlv_status status;
1537         int err, n_retry;
1538         char *in_mbox, *out_mbox, *tmp;
1539
1540         dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
1541                 reg->id, mlxsw_reg_id_str(reg->id),
1542                 mlxsw_core_reg_access_type_str(type));
1543
1544         in_mbox = mlxsw_cmd_mbox_alloc();
1545         if (!in_mbox)
1546                 return -ENOMEM;
1547
1548         out_mbox = mlxsw_cmd_mbox_alloc();
1549         if (!out_mbox) {
1550                 err = -ENOMEM;
1551                 goto free_in_mbox;
1552         }
1553
1554         mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
1555                                mlxsw_core_tid_get(mlxsw_core));
1556         tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
1557         mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
1558
1559         n_retry = 0;
1560 retry:
1561         err = mlxsw_cmd_access_reg(mlxsw_core, in_mbox, out_mbox);
1562         if (!err) {
1563                 err = mlxsw_emad_process_status(out_mbox, &status);
1564                 if (err) {
1565                         if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
1566                                 goto retry;
1567                         dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
1568                                 status, mlxsw_emad_op_tlv_status_str(status));
1569                 }
1570         }
1571
1572         if (!err)
1573                 memcpy(payload, mlxsw_emad_reg_payload(out_mbox),
1574                        reg->len);
1575
1576         mlxsw_cmd_mbox_free(out_mbox);
1577 free_in_mbox:
1578         mlxsw_cmd_mbox_free(in_mbox);
1579         if (err)
1580                 dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
1581                         reg->id, mlxsw_reg_id_str(reg->id),
1582                         mlxsw_core_reg_access_type_str(type));
1583         return err;
1584 }
1585
1586 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
1587                                      char *payload, size_t payload_len,
1588                                      unsigned long cb_priv)
1589 {
1590         char *orig_payload = (char *) cb_priv;
1591
1592         memcpy(orig_payload, payload, payload_len);
1593 }
1594
1595 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
1596                                  const struct mlxsw_reg_info *reg,
1597                                  char *payload,
1598                                  enum mlxsw_core_reg_access_type type)
1599 {
1600         LIST_HEAD(bulk_list);
1601         int err;
1602
1603         /* During initialization EMAD interface is not available to us,
1604          * so we default to command interface. We switch to EMAD interface
1605          * after setting the appropriate traps.
1606          */
1607         if (!mlxsw_core->emad.use_emad)
1608                 return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
1609                                                  payload, type);
1610
1611         err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
1612                                          payload, type, &bulk_list,
1613                                          mlxsw_core_reg_access_cb,
1614                                          (unsigned long) payload);
1615         if (err)
1616                 return err;
1617         return mlxsw_reg_trans_bulk_wait(&bulk_list);
1618 }
1619
1620 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
1621                     const struct mlxsw_reg_info *reg, char *payload)
1622 {
1623         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1624                                      MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
1625 }
1626 EXPORT_SYMBOL(mlxsw_reg_query);
1627
1628 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
1629                     const struct mlxsw_reg_info *reg, char *payload)
1630 {
1631         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1632                                      MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
1633 }
1634 EXPORT_SYMBOL(mlxsw_reg_write);
1635
1636 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1637                             struct mlxsw_rx_info *rx_info)
1638 {
1639         struct mlxsw_rx_listener_item *rxl_item;
1640         const struct mlxsw_rx_listener *rxl;
1641         struct mlxsw_core_pcpu_stats *pcpu_stats;
1642         u8 local_port;
1643         bool found = false;
1644
1645         if (rx_info->is_lag) {
1646                 dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
1647                                     __func__, rx_info->u.lag_id,
1648                                     rx_info->trap_id);
1649                 /* Upper layer does not care if the skb came from LAG or not,
1650                  * so just get the local_port for the lag port and push it up.
1651                  */
1652                 local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
1653                                                         rx_info->u.lag_id,
1654                                                         rx_info->lag_port_index);
1655         } else {
1656                 local_port = rx_info->u.sys_port;
1657         }
1658
1659         dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
1660                             __func__, local_port, rx_info->trap_id);
1661
1662         if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
1663             (local_port >= MLXSW_PORT_MAX_PORTS))
1664                 goto drop;
1665
1666         rcu_read_lock();
1667         list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
1668                 rxl = &rxl_item->rxl;
1669                 if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
1670                      rxl->local_port == local_port) &&
1671                     rxl->trap_id == rx_info->trap_id) {
1672                         found = true;
1673                         break;
1674                 }
1675         }
1676         rcu_read_unlock();
1677         if (!found)
1678                 goto drop;
1679
1680         pcpu_stats = this_cpu_ptr(mlxsw_core->pcpu_stats);
1681         u64_stats_update_begin(&pcpu_stats->syncp);
1682         pcpu_stats->port_rx_packets[local_port]++;
1683         pcpu_stats->port_rx_bytes[local_port] += skb->len;
1684         pcpu_stats->trap_rx_packets[rx_info->trap_id]++;
1685         pcpu_stats->trap_rx_bytes[rx_info->trap_id] += skb->len;
1686         u64_stats_update_end(&pcpu_stats->syncp);
1687
1688         rxl->func(skb, local_port, rxl_item->priv);
1689         return;
1690
1691 drop:
1692         if (rx_info->trap_id >= MLXSW_TRAP_ID_MAX)
1693                 this_cpu_inc(mlxsw_core->pcpu_stats->trap_rx_invalid);
1694         else
1695                 this_cpu_inc(mlxsw_core->pcpu_stats->trap_rx_dropped[rx_info->trap_id]);
1696         if (local_port >= MLXSW_PORT_MAX_PORTS)
1697                 this_cpu_inc(mlxsw_core->pcpu_stats->port_rx_invalid);
1698         else
1699                 this_cpu_inc(mlxsw_core->pcpu_stats->port_rx_dropped[local_port]);
1700         dev_kfree_skb(skb);
1701 }
1702 EXPORT_SYMBOL(mlxsw_core_skb_receive);
1703
1704 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
1705                                         u16 lag_id, u8 port_index)
1706 {
1707         return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
1708                port_index;
1709 }
1710
1711 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
1712                                 u16 lag_id, u8 port_index, u8 local_port)
1713 {
1714         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1715                                                  lag_id, port_index);
1716
1717         mlxsw_core->lag.mapping[index] = local_port;
1718 }
1719 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
1720
1721 u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
1722                               u16 lag_id, u8 port_index)
1723 {
1724         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1725                                                  lag_id, port_index);
1726
1727         return mlxsw_core->lag.mapping[index];
1728 }
1729 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
1730
1731 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
1732                                   u16 lag_id, u8 local_port)
1733 {
1734         int i;
1735
1736         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
1737                 int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1738                                                          lag_id, i);
1739
1740                 if (mlxsw_core->lag.mapping[index] == local_port)
1741                         mlxsw_core->lag.mapping[index] = 0;
1742         }
1743 }
1744 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
1745
1746 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
1747                           enum mlxsw_res_id res_id)
1748 {
1749         return mlxsw_res_valid(&mlxsw_core->res, res_id);
1750 }
1751 EXPORT_SYMBOL(mlxsw_core_res_valid);
1752
1753 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
1754                        enum mlxsw_res_id res_id)
1755 {
1756         return mlxsw_res_get(&mlxsw_core->res, res_id);
1757 }
1758 EXPORT_SYMBOL(mlxsw_core_res_get);
1759
1760 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port)
1761 {
1762         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1763         struct mlxsw_core_port *mlxsw_core_port =
1764                                         &mlxsw_core->ports[local_port];
1765         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1766         int err;
1767
1768         mlxsw_core_port->local_port = local_port;
1769         err = devlink_port_register(devlink, devlink_port, local_port);
1770         if (err)
1771                 memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1772         return err;
1773 }
1774 EXPORT_SYMBOL(mlxsw_core_port_init);
1775
1776 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
1777 {
1778         struct mlxsw_core_port *mlxsw_core_port =
1779                                         &mlxsw_core->ports[local_port];
1780         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1781
1782         devlink_port_unregister(devlink_port);
1783         memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1784 }
1785 EXPORT_SYMBOL(mlxsw_core_port_fini);
1786
1787 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1788                              void *port_driver_priv, struct net_device *dev,
1789                              bool split, u32 split_group)
1790 {
1791         struct mlxsw_core_port *mlxsw_core_port =
1792                                         &mlxsw_core->ports[local_port];
1793         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1794
1795         mlxsw_core_port->port_driver_priv = port_driver_priv;
1796         if (split)
1797                 devlink_port_split_set(devlink_port, split_group);
1798         devlink_port_type_eth_set(devlink_port, dev);
1799 }
1800 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
1801
1802 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1803                             void *port_driver_priv)
1804 {
1805         struct mlxsw_core_port *mlxsw_core_port =
1806                                         &mlxsw_core->ports[local_port];
1807         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1808
1809         mlxsw_core_port->port_driver_priv = port_driver_priv;
1810         devlink_port_type_ib_set(devlink_port, NULL);
1811 }
1812 EXPORT_SYMBOL(mlxsw_core_port_ib_set);
1813
1814 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
1815                            void *port_driver_priv)
1816 {
1817         struct mlxsw_core_port *mlxsw_core_port =
1818                                         &mlxsw_core->ports[local_port];
1819         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1820
1821         mlxsw_core_port->port_driver_priv = port_driver_priv;
1822         devlink_port_type_clear(devlink_port);
1823 }
1824 EXPORT_SYMBOL(mlxsw_core_port_clear);
1825
1826 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
1827                                                 u8 local_port)
1828 {
1829         struct mlxsw_core_port *mlxsw_core_port =
1830                                         &mlxsw_core->ports[local_port];
1831         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1832
1833         return devlink_port->type;
1834 }
1835 EXPORT_SYMBOL(mlxsw_core_port_type_get);
1836
1837 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
1838                                     const char *buf, size_t size)
1839 {
1840         __be32 *m = (__be32 *) buf;
1841         int i;
1842         int count = size / sizeof(__be32);
1843
1844         for (i = count - 1; i >= 0; i--)
1845                 if (m[i])
1846                         break;
1847         i++;
1848         count = i ? i : 1;
1849         for (i = 0; i < count; i += 4)
1850                 dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
1851                         i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
1852                         be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
1853 }
1854
1855 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
1856                    u32 in_mod, bool out_mbox_direct,
1857                    char *in_mbox, size_t in_mbox_size,
1858                    char *out_mbox, size_t out_mbox_size)
1859 {
1860         u8 status;
1861         int err;
1862
1863         BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
1864         if (!mlxsw_core->bus->cmd_exec)
1865                 return -EOPNOTSUPP;
1866
1867         dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
1868                 opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
1869         if (in_mbox) {
1870                 dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
1871                 mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
1872         }
1873
1874         err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
1875                                         opcode_mod, in_mod, out_mbox_direct,
1876                                         in_mbox, in_mbox_size,
1877                                         out_mbox, out_mbox_size, &status);
1878
1879         if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
1880                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
1881                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
1882                         in_mod, status, mlxsw_cmd_status_str(status));
1883         } else if (err == -ETIMEDOUT) {
1884                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
1885                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
1886                         in_mod);
1887         }
1888
1889         if (!err && out_mbox) {
1890                 dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
1891                 mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
1892         }
1893         return err;
1894 }
1895 EXPORT_SYMBOL(mlxsw_cmd_exec);
1896
1897 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
1898 {
1899         return queue_delayed_work(mlxsw_wq, dwork, delay);
1900 }
1901 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
1902
1903 static int __init mlxsw_core_module_init(void)
1904 {
1905         int err;
1906
1907         mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, WQ_MEM_RECLAIM, 0);
1908         if (!mlxsw_wq)
1909                 return -ENOMEM;
1910         mlxsw_core_dbg_root = debugfs_create_dir(mlxsw_core_driver_name, NULL);
1911         if (!mlxsw_core_dbg_root) {
1912                 err = -ENOMEM;
1913                 goto err_debugfs_create_dir;
1914         }
1915         return 0;
1916
1917 err_debugfs_create_dir:
1918         destroy_workqueue(mlxsw_wq);
1919         return err;
1920 }
1921
1922 static void __exit mlxsw_core_module_exit(void)
1923 {
1924         debugfs_remove_recursive(mlxsw_core_dbg_root);
1925         destroy_workqueue(mlxsw_wq);
1926 }
1927
1928 module_init(mlxsw_core_module_init);
1929 module_exit(mlxsw_core_module_exit);
1930
1931 MODULE_LICENSE("Dual BSD/GPL");
1932 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
1933 MODULE_DESCRIPTION("Mellanox switch device core driver");