drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30
  31 /* Local includes */
  32 #include "i40e.h"
  33 #include "i40e_diag.h"
  34 #include <net/udp_tunnel.h>
  35
  36 const char i40e_driver_name[] = "i40e";
  37 static const char i40e_driver_string[] =
  38                         "Intel(R) Ethernet Connection XL710 Network Driver";
  39
  40 #define DRV_KERN "-k"
  41
  42 #define DRV_VERSION_MAJOR 2
  43 #define DRV_VERSION_MINOR 1
  44 #define DRV_VERSION_BUILD 7
  45 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  46              __stringify(DRV_VERSION_MINOR) "." \
  47              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  48 const char i40e_driver_version_str[] = DRV_VERSION;
  49 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  50
  51 /* a bit of forward declarations */
  52 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  53 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  54 static int i40e_add_vsi(struct i40e_vsi *vsi);
  55 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  56 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  57 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  58 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  59 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  60 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
  61 static int i40e_reset(struct i40e_pf *pf);
  62 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  63 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  64 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  65
  66 /* i40e_pci_tbl - PCI Device ID Table
  67  *
  68  * Last entry must be all 0s
  69  *
  70  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  71  *   Class, Class Mask, private data (not used) }
  72  */
  73 static const struct pci_device_id i40e_pci_tbl[] = {
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  91         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  92         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  93         /* required last entry */
  94         {0, }
  95 };
  96 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  97
  98 #define I40E_MAX_VF_COUNT 128
  99 static int debug = -1;
 100 module_param(debug, uint, 0);
 101 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 102
 103 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 104 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 105 MODULE_LICENSE("GPL");
 106 MODULE_VERSION(DRV_VERSION);
 107
 108 static struct workqueue_struct *i40e_wq;
 109
 110 /**
 111  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 112  * @hw:   pointer to the HW structure
 113  * @mem:  ptr to mem struct to fill out
 114  * @size: size of memory requested
 115  * @alignment: what to align the allocation to
 116  **/
 117 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 118                             u64 size, u32 alignment)
 119 {
 120         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 121
 122         mem->size = ALIGN(size, alignment);
 123         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 124                                       &mem->pa, GFP_KERNEL);
 125         if (!mem->va)
 126                 return -ENOMEM;
 127
 128         return 0;
 129 }
 130
 131 /**
 132  * i40e_free_dma_mem_d - OS specific memory free for shared code
 133  * @hw:   pointer to the HW structure
 134  * @mem:  ptr to mem struct to free
 135  **/
 136 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 137 {
 138         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 139
 140         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 141         mem->va = NULL;
 142         mem->pa = 0;
 143         mem->size = 0;
 144
 145         return 0;
 146 }
 147
 148 /**
 149  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 150  * @hw:   pointer to the HW structure
 151  * @mem:  ptr to mem struct to fill out
 152  * @size: size of memory requested
 153  **/
 154 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 155                              u32 size)
 156 {
 157         mem->size = size;
 158         mem->va = kzalloc(size, GFP_KERNEL);
 159
 160         if (!mem->va)
 161                 return -ENOMEM;
 162
 163         return 0;
 164 }
 165
 166 /**
 167  * i40e_free_virt_mem_d - OS specific memory free for shared code
 168  * @hw:   pointer to the HW structure
 169  * @mem:  ptr to mem struct to free
 170  **/
 171 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 172 {
 173         /* it's ok to kfree a NULL pointer */
 174         kfree(mem->va);
 175         mem->va = NULL;
 176         mem->size = 0;
 177
 178         return 0;
 179 }
 180
 181 /**
 182  * i40e_get_lump - find a lump of free generic resource
 183  * @pf: board private structure
 184  * @pile: the pile of resource to search
 185  * @needed: the number of items needed
 186  * @id: an owner id to stick on the items assigned
 187  *
 188  * Returns the base item index of the lump, or negative for error
 189  *
 190  * The search_hint trick and lack of advanced fit-finding only work
 191  * because we're highly likely to have all the same size lump requests.
 192  * Linear search time and any fragmentation should be minimal.
 193  **/
 194 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 195                          u16 needed, u16 id)
 196 {
 197         int ret = -ENOMEM;
 198         int i, j;
 199
 200         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 201                 dev_info(&pf->pdev->dev,
 202                          "param err: pile=%p needed=%d id=0x%04x\n",
 203                          pile, needed, id);
 204                 return -EINVAL;
 205         }
 206
 207         /* start the linear search with an imperfect hint */
 208         i = pile->search_hint;
 209         while (i < pile->num_entries) {
 210                 /* skip already allocated entries */
 211                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 212                         i++;
 213                         continue;
 214                 }
 215
 216                 /* do we have enough in this lump? */
 217                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 218                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 219                                 break;
 220                 }
 221
 222                 if (j == needed) {
 223                         /* there was enough, so assign it to the requestor */
 224                         for (j = 0; j < needed; j++)
 225                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 226                         ret = i;
 227                         pile->search_hint = i + j;
 228                         break;
 229                 }
 230
 231                 /* not enough, so skip over it and continue looking */
 232                 i += j;
 233         }
 234
 235         return ret;
 236 }
 237
 238 /**
 239  * i40e_put_lump - return a lump of generic resource
 240  * @pile: the pile of resource to search
 241  * @index: the base item index
 242  * @id: the owner id of the items assigned
 243  *
 244  * Returns the count of items in the lump
 245  **/
 246 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 247 {
 248         int valid_id = (id | I40E_PILE_VALID_BIT);
 249         int count = 0;
 250         int i;
 251
 252         if (!pile || index >= pile->num_entries)
 253                 return -EINVAL;
 254
 255         for (i = index;
 256              i < pile->num_entries && pile->list[i] == valid_id;
 257              i++) {
 258                 pile->list[i] = 0;
 259                 count++;
 260         }
 261
 262         if (count && index < pile->search_hint)
 263                 pile->search_hint = index;
 264
 265         return count;
 266 }
 267
 268 /**
 269  * i40e_find_vsi_from_id - searches for the vsi with the given id
 270  * @pf - the pf structure to search for the vsi
 271  * @id - id of the vsi it is searching for
 272  **/
 273 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 274 {
 275         int i;
 276
 277         for (i = 0; i < pf->num_alloc_vsi; i++)
 278                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 279                         return pf->vsi[i];
 280
 281         return NULL;
 282 }
 283
 284 /**
 285  * i40e_service_event_schedule - Schedule the service task to wake up
 286  * @pf: board private structure
 287  *
 288  * If not already scheduled, this puts the task into the work queue
 289  **/
 290 void i40e_service_event_schedule(struct i40e_pf *pf)
 291 {
 292         if (!test_bit(__I40E_DOWN, &pf->state) &&
 293             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
 294                 queue_work(i40e_wq, &pf->service_task);
 295 }
 296
 297 /**
 298  * i40e_tx_timeout - Respond to a Tx Hang
 299  * @netdev: network interface device structure
 300  *
 301  * If any port has noticed a Tx timeout, it is likely that the whole
 302  * device is munged, not just the one netdev port, so go for the full
 303  * reset.
 304  **/
 305 static void i40e_tx_timeout(struct net_device *netdev)
 306 {
 307         struct i40e_netdev_priv *np = netdev_priv(netdev);
 308         struct i40e_vsi *vsi = np->vsi;
 309         struct i40e_pf *pf = vsi->back;
 310         struct i40e_ring *tx_ring = NULL;
 311         unsigned int i, hung_queue = 0;
 312         u32 head, val;
 313
 314         pf->tx_timeout_count++;
 315
 316         /* find the stopped queue the same way the stack does */
 317         for (i = 0; i < netdev->num_tx_queues; i++) {
 318                 struct netdev_queue *q;
 319                 unsigned long trans_start;
 320
 321                 q = netdev_get_tx_queue(netdev, i);
 322                 trans_start = q->trans_start;
 323                 if (netif_xmit_stopped(q) &&
 324                     time_after(jiffies,
 325                                (trans_start + netdev->watchdog_timeo))) {
 326                         hung_queue = i;
 327                         break;
 328                 }
 329         }
 330
 331         if (i == netdev->num_tx_queues) {
 332                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 333         } else {
 334                 /* now that we have an index, find the tx_ring struct */
 335                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 336                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 337                                 if (hung_queue ==
 338                                     vsi->tx_rings[i]->queue_index) {
 339                                         tx_ring = vsi->tx_rings[i];
 340                                         break;
 341                                 }
 342                         }
 343                 }
 344         }
 345
 346         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 347                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 348         else if (time_before(jiffies,
 349                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 350                 return;   /* don't do any new action before the next timeout */
 351
 352         if (tx_ring) {
 353                 head = i40e_get_head(tx_ring);
 354                 /* Read interrupt register */
 355                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 356                         val = rd32(&pf->hw,
 357                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 358                                                 tx_ring->vsi->base_vector - 1));
 359                 else
 360                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 361
 362                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 363                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 364                             head, tx_ring->next_to_use,
 365                             readl(tx_ring->tail), val);
 366         }
 367
 368         pf->tx_timeout_last_recovery = jiffies;
 369         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 370                     pf->tx_timeout_recovery_level, hung_queue);
 371
 372         switch (pf->tx_timeout_recovery_level) {
 373         case 1:
 374                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 375                 break;
 376         case 2:
 377                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 378                 break;
 379         case 3:
 380                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 381                 break;
 382         default:
 383                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 384                 break;
 385         }
 386
 387         i40e_service_event_schedule(pf);
 388         pf->tx_timeout_recovery_level++;
 389 }
 390
 391 /**
 392  * i40e_get_vsi_stats_struct - Get System Network Statistics
 393  * @vsi: the VSI we care about
 394  *
 395  * Returns the address of the device statistics structure.
 396  * The statistics are actually updated from the service task.
 397  **/
 398 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 399 {
 400         return &vsi->net_stats;
 401 }
 402
 403 /**
 404  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 405  * @netdev: network interface device structure
 406  *
 407  * Returns the address of the device statistics structure.
 408  * The statistics are actually updated from the service task.
 409  **/
 410 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 411                                   struct rtnl_link_stats64 *stats)
 412 {
 413         struct i40e_netdev_priv *np = netdev_priv(netdev);
 414         struct i40e_ring *tx_ring, *rx_ring;
 415         struct i40e_vsi *vsi = np->vsi;
 416         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 417         int i;
 418
 419         if (test_bit(__I40E_DOWN, &vsi->state))
 420                 return;
 421
 422         if (!vsi->tx_rings)
 423                 return;
 424
 425         rcu_read_lock();
 426         for (i = 0; i < vsi->num_queue_pairs; i++) {
 427                 u64 bytes, packets;
 428                 unsigned int start;
 429
 430                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 431                 if (!tx_ring)
 432                         continue;
 433
 434                 do {
 435                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
 436                         packets = tx_ring->stats.packets;
 437                         bytes   = tx_ring->stats.bytes;
 438                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
 439
 440                 stats->tx_packets += packets;
 441                 stats->tx_bytes   += bytes;
 442                 rx_ring = &tx_ring[1];
 443
 444                 do {
 445                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 446                         packets = rx_ring->stats.packets;
 447                         bytes   = rx_ring->stats.bytes;
 448                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 449
 450                 stats->rx_packets += packets;
 451                 stats->rx_bytes   += bytes;
 452         }
 453         rcu_read_unlock();
 454
 455         /* following stats updated by i40e_watchdog_subtask() */
 456         stats->multicast        = vsi_stats->multicast;
 457         stats->tx_errors        = vsi_stats->tx_errors;
 458         stats->tx_dropped       = vsi_stats->tx_dropped;
 459         stats->rx_errors        = vsi_stats->rx_errors;
 460         stats->rx_dropped       = vsi_stats->rx_dropped;
 461         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 462         stats->rx_length_errors = vsi_stats->rx_length_errors;
 463 }
 464
 465 /**
 466  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 467  * @vsi: the VSI to have its stats reset
 468  **/
 469 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 470 {
 471         struct rtnl_link_stats64 *ns;
 472         int i;
 473
 474         if (!vsi)
 475                 return;
 476
 477         ns = i40e_get_vsi_stats_struct(vsi);
 478         memset(ns, 0, sizeof(*ns));
 479         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 480         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 481         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 482         if (vsi->rx_rings && vsi->rx_rings[0]) {
 483                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 484                         memset(&vsi->rx_rings[i]->stats, 0,
 485                                sizeof(vsi->rx_rings[i]->stats));
 486                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 487                                sizeof(vsi->rx_rings[i]->rx_stats));
 488                         memset(&vsi->tx_rings[i]->stats, 0,
 489                                sizeof(vsi->tx_rings[i]->stats));
 490                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 491                                sizeof(vsi->tx_rings[i]->tx_stats));
 492                 }
 493         }
 494         vsi->stat_offsets_loaded = false;
 495 }
 496
 497 /**
 498  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 499  * @pf: the PF to be reset
 500  **/
 501 void i40e_pf_reset_stats(struct i40e_pf *pf)
 502 {
 503         int i;
 504
 505         memset(&pf->stats, 0, sizeof(pf->stats));
 506         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 507         pf->stat_offsets_loaded = false;
 508
 509         for (i = 0; i < I40E_MAX_VEB; i++) {
 510                 if (pf->veb[i]) {
 511                         memset(&pf->veb[i]->stats, 0,
 512                                sizeof(pf->veb[i]->stats));
 513                         memset(&pf->veb[i]->stats_offsets, 0,
 514                                sizeof(pf->veb[i]->stats_offsets));
 515                         pf->veb[i]->stat_offsets_loaded = false;
 516                 }
 517         }
 518         pf->hw_csum_rx_error = 0;
 519 }
 520
 521 /**
 522  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 523  * @hw: ptr to the hardware info
 524  * @hireg: the high 32 bit reg to read
 525  * @loreg: the low 32 bit reg to read
 526  * @offset_loaded: has the initial offset been loaded yet
 527  * @offset: ptr to current offset value
 528  * @stat: ptr to the stat
 529  *
 530  * Since the device stats are not reset at PFReset, they likely will not
 531  * be zeroed when the driver starts.  We'll save the first values read
 532  * and use them as offsets to be subtracted from the raw values in order
 533  * to report stats that count from zero.  In the process, we also manage
 534  * the potential roll-over.
 535  **/
 536 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 537                                bool offset_loaded, u64 *offset, u64 *stat)
 538 {
 539         u64 new_data;
 540
 541         if (hw->device_id == I40E_DEV_ID_QEMU) {
 542                 new_data = rd32(hw, loreg);
 543                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 544         } else {
 545                 new_data = rd64(hw, loreg);
 546         }
 547         if (!offset_loaded)
 548                 *offset = new_data;
 549         if (likely(new_data >= *offset))
 550                 *stat = new_data - *offset;
 551         else
 552                 *stat = (new_data + BIT_ULL(48)) - *offset;
 553         *stat &= 0xFFFFFFFFFFFFULL;
 554 }
 555
 556 /**
 557  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 558  * @hw: ptr to the hardware info
 559  * @reg: the hw reg to read
 560  * @offset_loaded: has the initial offset been loaded yet
 561  * @offset: ptr to current offset value
 562  * @stat: ptr to the stat
 563  **/
 564 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 565                                bool offset_loaded, u64 *offset, u64 *stat)
 566 {
 567         u32 new_data;
 568
 569         new_data = rd32(hw, reg);
 570         if (!offset_loaded)
 571                 *offset = new_data;
 572         if (likely(new_data >= *offset))
 573                 *stat = (u32)(new_data - *offset);
 574         else
 575                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 576 }
 577
 578 /**
 579  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 580  * @vsi: the VSI to be updated
 581  **/
 582 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 583 {
 584         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 585         struct i40e_pf *pf = vsi->back;
 586         struct i40e_hw *hw = &pf->hw;
 587         struct i40e_eth_stats *oes;
 588         struct i40e_eth_stats *es;     /* device's eth stats */
 589
 590         es = &vsi->eth_stats;
 591         oes = &vsi->eth_stats_offsets;
 592
 593         /* Gather up the stats that the hw collects */
 594         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 595                            vsi->stat_offsets_loaded,
 596                            &oes->tx_errors, &es->tx_errors);
 597         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 598                            vsi->stat_offsets_loaded,
 599                            &oes->rx_discards, &es->rx_discards);
 600         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 601                            vsi->stat_offsets_loaded,
 602                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 603         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 604                            vsi->stat_offsets_loaded,
 605                            &oes->tx_errors, &es->tx_errors);
 606
 607         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 608                            I40E_GLV_GORCL(stat_idx),
 609                            vsi->stat_offsets_loaded,
 610                            &oes->rx_bytes, &es->rx_bytes);
 611         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 612                            I40E_GLV_UPRCL(stat_idx),
 613                            vsi->stat_offsets_loaded,
 614                            &oes->rx_unicast, &es->rx_unicast);
 615         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 616                            I40E_GLV_MPRCL(stat_idx),
 617                            vsi->stat_offsets_loaded,
 618                            &oes->rx_multicast, &es->rx_multicast);
 619         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 620                            I40E_GLV_BPRCL(stat_idx),
 621                            vsi->stat_offsets_loaded,
 622                            &oes->rx_broadcast, &es->rx_broadcast);
 623
 624         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 625                            I40E_GLV_GOTCL(stat_idx),
 626                            vsi->stat_offsets_loaded,
 627                            &oes->tx_bytes, &es->tx_bytes);
 628         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 629                            I40E_GLV_UPTCL(stat_idx),
 630                            vsi->stat_offsets_loaded,
 631                            &oes->tx_unicast, &es->tx_unicast);
 632         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 633                            I40E_GLV_MPTCL(stat_idx),
 634                            vsi->stat_offsets_loaded,
 635                            &oes->tx_multicast, &es->tx_multicast);
 636         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 637                            I40E_GLV_BPTCL(stat_idx),
 638                            vsi->stat_offsets_loaded,
 639                            &oes->tx_broadcast, &es->tx_broadcast);
 640         vsi->stat_offsets_loaded = true;
 641 }
 642
 643 /**
 644  * i40e_update_veb_stats - Update Switch component statistics
 645  * @veb: the VEB being updated
 646  **/
 647 static void i40e_update_veb_stats(struct i40e_veb *veb)
 648 {
 649         struct i40e_pf *pf = veb->pf;
 650         struct i40e_hw *hw = &pf->hw;
 651         struct i40e_eth_stats *oes;
 652         struct i40e_eth_stats *es;     /* device's eth stats */
 653         struct i40e_veb_tc_stats *veb_oes;
 654         struct i40e_veb_tc_stats *veb_es;
 655         int i, idx = 0;
 656
 657         idx = veb->stats_idx;
 658         es = &veb->stats;
 659         oes = &veb->stats_offsets;
 660         veb_es = &veb->tc_stats;
 661         veb_oes = &veb->tc_stats_offsets;
 662
 663         /* Gather up the stats that the hw collects */
 664         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 665                            veb->stat_offsets_loaded,
 666                            &oes->tx_discards, &es->tx_discards);
 667         if (hw->revision_id > 0)
 668                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 669                                    veb->stat_offsets_loaded,
 670                                    &oes->rx_unknown_protocol,
 671                                    &es->rx_unknown_protocol);
 672         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 673                            veb->stat_offsets_loaded,
 674                            &oes->rx_bytes, &es->rx_bytes);
 675         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 676                            veb->stat_offsets_loaded,
 677                            &oes->rx_unicast, &es->rx_unicast);
 678         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 679                            veb->stat_offsets_loaded,
 680                            &oes->rx_multicast, &es->rx_multicast);
 681         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 682                            veb->stat_offsets_loaded,
 683                            &oes->rx_broadcast, &es->rx_broadcast);
 684
 685         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 686                            veb->stat_offsets_loaded,
 687                            &oes->tx_bytes, &es->tx_bytes);
 688         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 689                            veb->stat_offsets_loaded,
 690                            &oes->tx_unicast, &es->tx_unicast);
 691         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 692                            veb->stat_offsets_loaded,
 693                            &oes->tx_multicast, &es->tx_multicast);
 694         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 695                            veb->stat_offsets_loaded,
 696                            &oes->tx_broadcast, &es->tx_broadcast);
 697         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 698                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 699                                    I40E_GLVEBTC_RPCL(i, idx),
 700                                    veb->stat_offsets_loaded,
 701                                    &veb_oes->tc_rx_packets[i],
 702                                    &veb_es->tc_rx_packets[i]);
 703                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 704                                    I40E_GLVEBTC_RBCL(i, idx),
 705                                    veb->stat_offsets_loaded,
 706                                    &veb_oes->tc_rx_bytes[i],
 707                                    &veb_es->tc_rx_bytes[i]);
 708                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 709                                    I40E_GLVEBTC_TPCL(i, idx),
 710                                    veb->stat_offsets_loaded,
 711                                    &veb_oes->tc_tx_packets[i],
 712                                    &veb_es->tc_tx_packets[i]);
 713                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 714                                    I40E_GLVEBTC_TBCL(i, idx),
 715                                    veb->stat_offsets_loaded,
 716                                    &veb_oes->tc_tx_bytes[i],
 717                                    &veb_es->tc_tx_bytes[i]);
 718         }
 719         veb->stat_offsets_loaded = true;
 720 }
 721
 722 /**
 723  * i40e_update_vsi_stats - Update the vsi statistics counters.
 724  * @vsi: the VSI to be updated
 725  *
 726  * There are a few instances where we store the same stat in a
 727  * couple of different structs.  This is partly because we have
 728  * the netdev stats that need to be filled out, which is slightly
 729  * different from the "eth_stats" defined by the chip and used in
 730  * VF communications.  We sort it out here.
 731  **/
 732 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 733 {
 734         struct i40e_pf *pf = vsi->back;
 735         struct rtnl_link_stats64 *ons;
 736         struct rtnl_link_stats64 *ns;   /* netdev stats */
 737         struct i40e_eth_stats *oes;
 738         struct i40e_eth_stats *es;     /* device's eth stats */
 739         u32 tx_restart, tx_busy;
 740         struct i40e_ring *p;
 741         u32 rx_page, rx_buf;
 742         u64 bytes, packets;
 743         unsigned int start;
 744         u64 tx_linearize;
 745         u64 tx_force_wb;
 746         u64 rx_p, rx_b;
 747         u64 tx_p, tx_b;
 748         u16 q;
 749
 750         if (test_bit(__I40E_DOWN, &vsi->state) ||
 751             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 752                 return;
 753
 754         ns = i40e_get_vsi_stats_struct(vsi);
 755         ons = &vsi->net_stats_offsets;
 756         es = &vsi->eth_stats;
 757         oes = &vsi->eth_stats_offsets;
 758
 759         /* Gather up the netdev and vsi stats that the driver collects
 760          * on the fly during packet processing
 761          */
 762         rx_b = rx_p = 0;
 763         tx_b = tx_p = 0;
 764         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 765         rx_page = 0;
 766         rx_buf = 0;
 767         rcu_read_lock();
 768         for (q = 0; q < vsi->num_queue_pairs; q++) {
 769                 /* locate Tx ring */
 770                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 771
 772                 do {
 773                         start = u64_stats_fetch_begin_irq(&p->syncp);
 774                         packets = p->stats.packets;
 775                         bytes = p->stats.bytes;
 776                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 777                 tx_b += bytes;
 778                 tx_p += packets;
 779                 tx_restart += p->tx_stats.restart_queue;
 780                 tx_busy += p->tx_stats.tx_busy;
 781                 tx_linearize += p->tx_stats.tx_linearize;
 782                 tx_force_wb += p->tx_stats.tx_force_wb;
 783
 784                 /* Rx queue is part of the same block as Tx queue */
 785                 p = &p[1];
 786                 do {
 787                         start = u64_stats_fetch_begin_irq(&p->syncp);
 788                         packets = p->stats.packets;
 789                         bytes = p->stats.bytes;
 790                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 791                 rx_b += bytes;
 792                 rx_p += packets;
 793                 rx_buf += p->rx_stats.alloc_buff_failed;
 794                 rx_page += p->rx_stats.alloc_page_failed;
 795         }
 796         rcu_read_unlock();
 797         vsi->tx_restart = tx_restart;
 798         vsi->tx_busy = tx_busy;
 799         vsi->tx_linearize = tx_linearize;
 800         vsi->tx_force_wb = tx_force_wb;
 801         vsi->rx_page_failed = rx_page;
 802         vsi->rx_buf_failed = rx_buf;
 803
 804         ns->rx_packets = rx_p;
 805         ns->rx_bytes = rx_b;
 806         ns->tx_packets = tx_p;
 807         ns->tx_bytes = tx_b;
 808
 809         /* update netdev stats from eth stats */
 810         i40e_update_eth_stats(vsi);
 811         ons->tx_errors = oes->tx_errors;
 812         ns->tx_errors = es->tx_errors;
 813         ons->multicast = oes->rx_multicast;
 814         ns->multicast = es->rx_multicast;
 815         ons->rx_dropped = oes->rx_discards;
 816         ns->rx_dropped = es->rx_discards;
 817         ons->tx_dropped = oes->tx_discards;
 818         ns->tx_dropped = es->tx_discards;
 819
 820         /* pull in a couple PF stats if this is the main vsi */
 821         if (vsi == pf->vsi[pf->lan_vsi]) {
 822                 ns->rx_crc_errors = pf->stats.crc_errors;
 823                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 824                 ns->rx_length_errors = pf->stats.rx_length_errors;
 825         }
 826 }
 827
 828 /**
 829  * i40e_update_pf_stats - Update the PF statistics counters.
 830  * @pf: the PF to be updated
 831  **/
 832 static void i40e_update_pf_stats(struct i40e_pf *pf)
 833 {
 834         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 835         struct i40e_hw_port_stats *nsd = &pf->stats;
 836         struct i40e_hw *hw = &pf->hw;
 837         u32 val;
 838         int i;
 839
 840         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 841                            I40E_GLPRT_GORCL(hw->port),
 842                            pf->stat_offsets_loaded,
 843                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 844         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 845                            I40E_GLPRT_GOTCL(hw->port),
 846                            pf->stat_offsets_loaded,
 847                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 848         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 849                            pf->stat_offsets_loaded,
 850                            &osd->eth.rx_discards,
 851                            &nsd->eth.rx_discards);
 852         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 853                            I40E_GLPRT_UPRCL(hw->port),
 854                            pf->stat_offsets_loaded,
 855                            &osd->eth.rx_unicast,
 856                            &nsd->eth.rx_unicast);
 857         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 858                            I40E_GLPRT_MPRCL(hw->port),
 859                            pf->stat_offsets_loaded,
 860                            &osd->eth.rx_multicast,
 861                            &nsd->eth.rx_multicast);
 862         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 863                            I40E_GLPRT_BPRCL(hw->port),
 864                            pf->stat_offsets_loaded,
 865                            &osd->eth.rx_broadcast,
 866                            &nsd->eth.rx_broadcast);
 867         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 868                            I40E_GLPRT_UPTCL(hw->port),
 869                            pf->stat_offsets_loaded,
 870                            &osd->eth.tx_unicast,
 871                            &nsd->eth.tx_unicast);
 872         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 873                            I40E_GLPRT_MPTCL(hw->port),
 874                            pf->stat_offsets_loaded,
 875                            &osd->eth.tx_multicast,
 876                            &nsd->eth.tx_multicast);
 877         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 878                            I40E_GLPRT_BPTCL(hw->port),
 879                            pf->stat_offsets_loaded,
 880                            &osd->eth.tx_broadcast,
 881                            &nsd->eth.tx_broadcast);
 882
 883         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 884                            pf->stat_offsets_loaded,
 885                            &osd->tx_dropped_link_down,
 886                            &nsd->tx_dropped_link_down);
 887
 888         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 889                            pf->stat_offsets_loaded,
 890                            &osd->crc_errors, &nsd->crc_errors);
 891
 892         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 893                            pf->stat_offsets_loaded,
 894                            &osd->illegal_bytes, &nsd->illegal_bytes);
 895
 896         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 897                            pf->stat_offsets_loaded,
 898                            &osd->mac_local_faults,
 899                            &nsd->mac_local_faults);
 900         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 901                            pf->stat_offsets_loaded,
 902                            &osd->mac_remote_faults,
 903                            &nsd->mac_remote_faults);
 904
 905         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 906                            pf->stat_offsets_loaded,
 907                            &osd->rx_length_errors,
 908                            &nsd->rx_length_errors);
 909
 910         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 911                            pf->stat_offsets_loaded,
 912                            &osd->link_xon_rx, &nsd->link_xon_rx);
 913         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 914                            pf->stat_offsets_loaded,
 915                            &osd->link_xon_tx, &nsd->link_xon_tx);
 916         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 919         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 920                            pf->stat_offsets_loaded,
 921                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 922
 923         for (i = 0; i < 8; i++) {
 924                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 925                                    pf->stat_offsets_loaded,
 926                                    &osd->priority_xoff_rx[i],
 927                                    &nsd->priority_xoff_rx[i]);
 928                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 929                                    pf->stat_offsets_loaded,
 930                                    &osd->priority_xon_rx[i],
 931                                    &nsd->priority_xon_rx[i]);
 932                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 933                                    pf->stat_offsets_loaded,
 934                                    &osd->priority_xon_tx[i],
 935                                    &nsd->priority_xon_tx[i]);
 936                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 937                                    pf->stat_offsets_loaded,
 938                                    &osd->priority_xoff_tx[i],
 939                                    &nsd->priority_xoff_tx[i]);
 940                 i40e_stat_update32(hw,
 941                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 942                                    pf->stat_offsets_loaded,
 943                                    &osd->priority_xon_2_xoff[i],
 944                                    &nsd->priority_xon_2_xoff[i]);
 945         }
 946
 947         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 948                            I40E_GLPRT_PRC64L(hw->port),
 949                            pf->stat_offsets_loaded,
 950                            &osd->rx_size_64, &nsd->rx_size_64);
 951         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 952                            I40E_GLPRT_PRC127L(hw->port),
 953                            pf->stat_offsets_loaded,
 954                            &osd->rx_size_127, &nsd->rx_size_127);
 955         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 956                            I40E_GLPRT_PRC255L(hw->port),
 957                            pf->stat_offsets_loaded,
 958                            &osd->rx_size_255, &nsd->rx_size_255);
 959         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
 960                            I40E_GLPRT_PRC511L(hw->port),
 961                            pf->stat_offsets_loaded,
 962                            &osd->rx_size_511, &nsd->rx_size_511);
 963         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
 964                            I40E_GLPRT_PRC1023L(hw->port),
 965                            pf->stat_offsets_loaded,
 966                            &osd->rx_size_1023, &nsd->rx_size_1023);
 967         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
 968                            I40E_GLPRT_PRC1522L(hw->port),
 969                            pf->stat_offsets_loaded,
 970                            &osd->rx_size_1522, &nsd->rx_size_1522);
 971         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
 972                            I40E_GLPRT_PRC9522L(hw->port),
 973                            pf->stat_offsets_loaded,
 974                            &osd->rx_size_big, &nsd->rx_size_big);
 975
 976         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
 977                            I40E_GLPRT_PTC64L(hw->port),
 978                            pf->stat_offsets_loaded,
 979                            &osd->tx_size_64, &nsd->tx_size_64);
 980         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
 981                            I40E_GLPRT_PTC127L(hw->port),
 982                            pf->stat_offsets_loaded,
 983                            &osd->tx_size_127, &nsd->tx_size_127);
 984         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
 985                            I40E_GLPRT_PTC255L(hw->port),
 986                            pf->stat_offsets_loaded,
 987                            &osd->tx_size_255, &nsd->tx_size_255);
 988         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
 989                            I40E_GLPRT_PTC511L(hw->port),
 990                            pf->stat_offsets_loaded,
 991                            &osd->tx_size_511, &nsd->tx_size_511);
 992         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
 993                            I40E_GLPRT_PTC1023L(hw->port),
 994                            pf->stat_offsets_loaded,
 995                            &osd->tx_size_1023, &nsd->tx_size_1023);
 996         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
 997                            I40E_GLPRT_PTC1522L(hw->port),
 998                            pf->stat_offsets_loaded,
 999                            &osd->tx_size_1522, &nsd->tx_size_1522);
1000         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1001                            I40E_GLPRT_PTC9522L(hw->port),
1002                            pf->stat_offsets_loaded,
1003                            &osd->tx_size_big, &nsd->tx_size_big);
1004
1005         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1006                            pf->stat_offsets_loaded,
1007                            &osd->rx_undersize, &nsd->rx_undersize);
1008         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1009                            pf->stat_offsets_loaded,
1010                            &osd->rx_fragments, &nsd->rx_fragments);
1011         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1012                            pf->stat_offsets_loaded,
1013                            &osd->rx_oversize, &nsd->rx_oversize);
1014         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1015                            pf->stat_offsets_loaded,
1016                            &osd->rx_jabber, &nsd->rx_jabber);
1017
1018         /* FDIR stats */
1019         i40e_stat_update32(hw,
1020                            I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)),
1021                            pf->stat_offsets_loaded,
1022                            &osd->fd_atr_match, &nsd->fd_atr_match);
1023         i40e_stat_update32(hw,
1024                            I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)),
1025                            pf->stat_offsets_loaded,
1026                            &osd->fd_sb_match, &nsd->fd_sb_match);
1027         i40e_stat_update32(hw,
1028                       I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)),
1029                       pf->stat_offsets_loaded,
1030                       &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match);
1031
1032         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1033         nsd->tx_lpi_status =
1034                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1035                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1036         nsd->rx_lpi_status =
1037                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1038                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1039         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1042         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1043                            pf->stat_offsets_loaded,
1044                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1045
1046         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1047             !(pf->hw_disabled_flags & I40E_FLAG_FD_SB_ENABLED))
1048                 nsd->fd_sb_status = true;
1049         else
1050                 nsd->fd_sb_status = false;
1051
1052         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1053             !(pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED))
1054                 nsd->fd_atr_status = true;
1055         else
1056                 nsd->fd_atr_status = false;
1057
1058         pf->stat_offsets_loaded = true;
1059 }
1060
1061 /**
1062  * i40e_update_stats - Update the various statistics counters.
1063  * @vsi: the VSI to be updated
1064  *
1065  * Update the various stats for this VSI and its related entities.
1066  **/
1067 void i40e_update_stats(struct i40e_vsi *vsi)
1068 {
1069         struct i40e_pf *pf = vsi->back;
1070
1071         if (vsi == pf->vsi[pf->lan_vsi])
1072                 i40e_update_pf_stats(pf);
1073
1074         i40e_update_vsi_stats(vsi);
1075 }
1076
1077 /**
1078  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1079  * @vsi: the VSI to be searched
1080  * @macaddr: the MAC address
1081  * @vlan: the vlan
1082  *
1083  * Returns ptr to the filter object or NULL
1084  **/
1085 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1086                                                 const u8 *macaddr, s16 vlan)
1087 {
1088         struct i40e_mac_filter *f;
1089         u64 key;
1090
1091         if (!vsi || !macaddr)
1092                 return NULL;
1093
1094         key = i40e_addr_to_hkey(macaddr);
1095         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1096                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1097                     (vlan == f->vlan))
1098                         return f;
1099         }
1100         return NULL;
1101 }
1102
1103 /**
1104  * i40e_find_mac - Find a mac addr in the macvlan filters list
1105  * @vsi: the VSI to be searched
1106  * @macaddr: the MAC address we are searching for
1107  *
1108  * Returns the first filter with the provided MAC address or NULL if
1109  * MAC address was not found
1110  **/
1111 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1112 {
1113         struct i40e_mac_filter *f;
1114         u64 key;
1115
1116         if (!vsi || !macaddr)
1117                 return NULL;
1118
1119         key = i40e_addr_to_hkey(macaddr);
1120         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1121                 if ((ether_addr_equal(macaddr, f->macaddr)))
1122                         return f;
1123         }
1124         return NULL;
1125 }
1126
1127 /**
1128  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1129  * @vsi: the VSI to be searched
1130  *
1131  * Returns true if VSI is in vlan mode or false otherwise
1132  **/
1133 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1134 {
1135         /* If we have a PVID, always operate in VLAN mode */
1136         if (vsi->info.pvid)
1137                 return true;
1138
1139         /* We need to operate in VLAN mode whenever we have any filters with
1140          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1141          * time, incurring search cost repeatedly. However, we can notice two
1142          * things:
1143          *
1144          * 1) the only place where we can gain a VLAN filter is in
1145          *    i40e_add_filter.
1146          *
1147          * 2) the only place where filters are actually removed is in
1148          *    i40e_sync_filters_subtask.
1149          *
1150          * Thus, we can simply use a boolean value, has_vlan_filters which we
1151          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1152          * we have to perform the full search after deleting filters in
1153          * i40e_sync_filters_subtask, but we already have to search
1154          * filters here and can perform the check at the same time. This
1155          * results in avoiding embedding a loop for VLAN mode inside another
1156          * loop over all the filters, and should maintain correctness as noted
1157          * above.
1158          */
1159         return vsi->has_vlan_filter;
1160 }
1161
1162 /**
1163  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1164  * @vsi: the VSI to configure
1165  * @tmp_add_list: list of filters ready to be added
1166  * @tmp_del_list: list of filters ready to be deleted
1167  * @vlan_filters: the number of active VLAN filters
1168  *
1169  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1170  * behave as expected. If we have any active VLAN filters remaining or about
1171  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1172  * so that they only match against untagged traffic. If we no longer have any
1173  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1174  * so that they match against both tagged and untagged traffic. In this way,
1175  * we ensure that we correctly receive the desired traffic. This ensures that
1176  * when we have an active VLAN we will receive only untagged traffic and
1177  * traffic matching active VLANs. If we have no active VLANs then we will
1178  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1179  *
1180  * Finally, in a similar fashion, this function also corrects filters when
1181  * there is an active PVID assigned to this VSI.
1182  *
1183  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1184  *
1185  * This function is only expected to be called from within
1186  * i40e_sync_vsi_filters.
1187  *
1188  * NOTE: This function expects to be called while under the
1189  * mac_filter_hash_lock
1190  */
1191 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1192                                          struct hlist_head *tmp_add_list,
1193                                          struct hlist_head *tmp_del_list,
1194                                          int vlan_filters)
1195 {
1196         s16 pvid = le16_to_cpu(vsi->info.pvid);
1197         struct i40e_mac_filter *f, *add_head;
1198         struct i40e_new_mac_filter *new;
1199         struct hlist_node *h;
1200         int bkt, new_vlan;
1201
1202         /* To determine if a particular filter needs to be replaced we
1203          * have the three following conditions:
1204          *
1205          * a) if we have a PVID assigned, then all filters which are
1206          *    not marked as VLAN=PVID must be replaced with filters that
1207          *    are.
1208          * b) otherwise, if we have any active VLANS, all filters
1209          *    which are marked as VLAN=-1 must be replaced with
1210          *    filters marked as VLAN=0
1211          * c) finally, if we do not have any active VLANS, all filters
1212          *    which are marked as VLAN=0 must be replaced with filters
1213          *    marked as VLAN=-1
1214          */
1215
1216         /* Update the filters about to be added in place */
1217         hlist_for_each_entry(new, tmp_add_list, hlist) {
1218                 if (pvid && new->f->vlan != pvid)
1219                         new->f->vlan = pvid;
1220                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1221                         new->f->vlan = 0;
1222                 else if (!vlan_filters && new->f->vlan == 0)
1223                         new->f->vlan = I40E_VLAN_ANY;
1224         }
1225
1226         /* Update the remaining active filters */
1227         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1228                 /* Combine the checks for whether a filter needs to be changed
1229                  * and then determine the new VLAN inside the if block, in
1230                  * order to avoid duplicating code for adding the new filter
1231                  * then deleting the old filter.
1232                  */
1233                 if ((pvid && f->vlan != pvid) ||
1234                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1235                     (!vlan_filters && f->vlan == 0)) {
1236                         /* Determine the new vlan we will be adding */
1237                         if (pvid)
1238                                 new_vlan = pvid;
1239                         else if (vlan_filters)
1240                                 new_vlan = 0;
1241                         else
1242                                 new_vlan = I40E_VLAN_ANY;
1243
1244                         /* Create the new filter */
1245                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1246                         if (!add_head)
1247                                 return -ENOMEM;
1248
1249                         /* Create a temporary i40e_new_mac_filter */
1250                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1251                         if (!new)
1252                                 return -ENOMEM;
1253
1254                         new->f = add_head;
1255                         new->state = add_head->state;
1256
1257                         /* Add the new filter to the tmp list */
1258                         hlist_add_head(&new->hlist, tmp_add_list);
1259
1260                         /* Put the original filter into the delete list */
1261                         f->state = I40E_FILTER_REMOVE;
1262                         hash_del(&f->hlist);
1263                         hlist_add_head(&f->hlist, tmp_del_list);
1264                 }
1265         }
1266
1267         vsi->has_vlan_filter = !!vlan_filters;
1268
1269         return 0;
1270 }
1271
1272 /**
1273  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1274  * @vsi: the PF Main VSI - inappropriate for any other VSI
1275  * @macaddr: the MAC address
1276  *
1277  * Remove whatever filter the firmware set up so the driver can manage
1278  * its own filtering intelligently.
1279  **/
1280 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1281 {
1282         struct i40e_aqc_remove_macvlan_element_data element;
1283         struct i40e_pf *pf = vsi->back;
1284
1285         /* Only appropriate for the PF main VSI */
1286         if (vsi->type != I40E_VSI_MAIN)
1287                 return;
1288
1289         memset(&element, 0, sizeof(element));
1290         ether_addr_copy(element.mac_addr, macaddr);
1291         element.vlan_tag = 0;
1292         /* Ignore error returns, some firmware does it this way... */
1293         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1294         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1295
1296         memset(&element, 0, sizeof(element));
1297         ether_addr_copy(element.mac_addr, macaddr);
1298         element.vlan_tag = 0;
1299         /* ...and some firmware does it this way. */
1300         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1301                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1302         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1303 }
1304
1305 /**
1306  * i40e_add_filter - Add a mac/vlan filter to the VSI
1307  * @vsi: the VSI to be searched
1308  * @macaddr: the MAC address
1309  * @vlan: the vlan
1310  *
1311  * Returns ptr to the filter object or NULL when no memory available.
1312  *
1313  * NOTE: This function is expected to be called with mac_filter_hash_lock
1314  * being held.
1315  **/
1316 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1317                                         const u8 *macaddr, s16 vlan)
1318 {
1319         struct i40e_mac_filter *f;
1320         u64 key;
1321
1322         if (!vsi || !macaddr)
1323                 return NULL;
1324
1325         f = i40e_find_filter(vsi, macaddr, vlan);
1326         if (!f) {
1327                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1328                 if (!f)
1329                         return NULL;
1330
1331                 /* Update the boolean indicating if we need to function in
1332                  * VLAN mode.
1333                  */
1334                 if (vlan >= 0)
1335                         vsi->has_vlan_filter = true;
1336
1337                 ether_addr_copy(f->macaddr, macaddr);
1338                 f->vlan = vlan;
1339                 /* If we're in overflow promisc mode, set the state directly
1340                  * to failed, so we don't bother to try sending the filter
1341                  * to the hardware.
1342                  */
1343                 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))
1344                         f->state = I40E_FILTER_FAILED;
1345                 else
1346                         f->state = I40E_FILTER_NEW;
1347                 INIT_HLIST_NODE(&f->hlist);
1348
1349                 key = i40e_addr_to_hkey(macaddr);
1350                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1351
1352                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1353                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1354         }
1355
1356         /* If we're asked to add a filter that has been marked for removal, it
1357          * is safe to simply restore it to active state. __i40e_del_filter
1358          * will have simply deleted any filters which were previously marked
1359          * NEW or FAILED, so if it is currently marked REMOVE it must have
1360          * previously been ACTIVE. Since we haven't yet run the sync filters
1361          * task, just restore this filter to the ACTIVE state so that the
1362          * sync task leaves it in place
1363          */
1364         if (f->state == I40E_FILTER_REMOVE)
1365                 f->state = I40E_FILTER_ACTIVE;
1366
1367         return f;
1368 }
1369
1370 /**
1371  * __i40e_del_filter - Remove a specific filter from the VSI
1372  * @vsi: VSI to remove from
1373  * @f: the filter to remove from the list
1374  *
1375  * This function should be called instead of i40e_del_filter only if you know
1376  * the exact filter you will remove already, such as via i40e_find_filter or
1377  * i40e_find_mac.
1378  *
1379  * NOTE: This function is expected to be called with mac_filter_hash_lock
1380  * being held.
1381  * ANOTHER NOTE: This function MUST be called from within the context of
1382  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1383  * instead of list_for_each_entry().
1384  **/
1385 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1386 {
1387         if (!f)
1388                 return;
1389
1390         /* If the filter was never added to firmware then we can just delete it
1391          * directly and we don't want to set the status to remove or else an
1392          * admin queue command will unnecessarily fire.
1393          */
1394         if ((f->state == I40E_FILTER_FAILED) ||
1395             (f->state == I40E_FILTER_NEW)) {
1396                 hash_del(&f->hlist);
1397                 kfree(f);
1398         } else {
1399                 f->state = I40E_FILTER_REMOVE;
1400         }
1401
1402         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1403         vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1404 }
1405
1406 /**
1407  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1408  * @vsi: the VSI to be searched
1409  * @macaddr: the MAC address
1410  * @vlan: the VLAN
1411  *
1412  * NOTE: This function is expected to be called with mac_filter_hash_lock
1413  * being held.
1414  * ANOTHER NOTE: This function MUST be called from within the context of
1415  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1416  * instead of list_for_each_entry().
1417  **/
1418 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1419 {
1420         struct i40e_mac_filter *f;
1421
1422         if (!vsi || !macaddr)
1423                 return;
1424
1425         f = i40e_find_filter(vsi, macaddr, vlan);
1426         __i40e_del_filter(vsi, f);
1427 }
1428
1429 /**
1430  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1431  * @vsi: the VSI to be searched
1432  * @macaddr: the mac address to be filtered
1433  *
1434  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1435  * go through all the macvlan filters and add a macvlan filter for each
1436  * unique vlan that already exists. If a PVID has been assigned, instead only
1437  * add the macaddr to that VLAN.
1438  *
1439  * Returns last filter added on success, else NULL
1440  **/
1441 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1442                                             const u8 *macaddr)
1443 {
1444         struct i40e_mac_filter *f, *add = NULL;
1445         struct hlist_node *h;
1446         int bkt;
1447
1448         if (vsi->info.pvid)
1449                 return i40e_add_filter(vsi, macaddr,
1450                                        le16_to_cpu(vsi->info.pvid));
1451
1452         if (!i40e_is_vsi_in_vlan(vsi))
1453                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1454
1455         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1456                 if (f->state == I40E_FILTER_REMOVE)
1457                         continue;
1458                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1459                 if (!add)
1460                         return NULL;
1461         }
1462
1463         return add;
1464 }
1465
1466 /**
1467  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1468  * @vsi: the VSI to be searched
1469  * @macaddr: the mac address to be removed
1470  *
1471  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1472  * associated with.
1473  *
1474  * Returns 0 for success, or error
1475  **/
1476 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1477 {
1478         struct i40e_mac_filter *f;
1479         struct hlist_node *h;
1480         bool found = false;
1481         int bkt;
1482
1483         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1484              "Missing mac_filter_hash_lock\n");
1485         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1486                 if (ether_addr_equal(macaddr, f->macaddr)) {
1487                         __i40e_del_filter(vsi, f);
1488                         found = true;
1489                 }
1490         }
1491
1492         if (found)
1493                 return 0;
1494         else
1495                 return -ENOENT;
1496 }
1497
1498 /**
1499  * i40e_set_mac - NDO callback to set mac address
1500  * @netdev: network interface device structure
1501  * @p: pointer to an address structure
1502  *
1503  * Returns 0 on success, negative on failure
1504  **/
1505 static int i40e_set_mac(struct net_device *netdev, void *p)
1506 {
1507         struct i40e_netdev_priv *np = netdev_priv(netdev);
1508         struct i40e_vsi *vsi = np->vsi;
1509         struct i40e_pf *pf = vsi->back;
1510         struct i40e_hw *hw = &pf->hw;
1511         struct sockaddr *addr = p;
1512
1513         if (!is_valid_ether_addr(addr->sa_data))
1514                 return -EADDRNOTAVAIL;
1515
1516         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1517                 netdev_info(netdev, "already using mac address %pM\n",
1518                             addr->sa_data);
1519                 return 0;
1520         }
1521
1522         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1523             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1524                 return -EADDRNOTAVAIL;
1525
1526         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1527                 netdev_info(netdev, "returning to hw mac address %pM\n",
1528                             hw->mac.addr);
1529         else
1530                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1531
1532         spin_lock_bh(&vsi->mac_filter_hash_lock);
1533         i40e_del_mac_filter(vsi, netdev->dev_addr);
1534         i40e_add_mac_filter(vsi, addr->sa_data);
1535         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1536         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1537         if (vsi->type == I40E_VSI_MAIN) {
1538                 i40e_status ret;
1539
1540                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1541                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1542                                                 addr->sa_data, NULL);
1543                 if (ret)
1544                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1545                                     i40e_stat_str(hw, ret),
1546                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1547         }
1548
1549         /* schedule our worker thread which will take care of
1550          * applying the new filter changes
1551          */
1552         i40e_service_event_schedule(vsi->back);
1553         return 0;
1554 }
1555
1556 /**
1557  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1558  * @vsi: the VSI being setup
1559  * @ctxt: VSI context structure
1560  * @enabled_tc: Enabled TCs bitmap
1561  * @is_add: True if called before Add VSI
1562  *
1563  * Setup VSI queue mapping for enabled traffic classes.
1564  **/
1565 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1566                                      struct i40e_vsi_context *ctxt,
1567                                      u8 enabled_tc,
1568                                      bool is_add)
1569 {
1570         struct i40e_pf *pf = vsi->back;
1571         u16 sections = 0;
1572         u8 netdev_tc = 0;
1573         u16 numtc = 0;
1574         u16 qcount;
1575         u8 offset;
1576         u16 qmap;
1577         int i;
1578         u16 num_tc_qps = 0;
1579
1580         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1581         offset = 0;
1582
1583         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1584                 /* Find numtc from enabled TC bitmap */
1585                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1586                         if (enabled_tc & BIT(i)) /* TC is enabled */
1587                                 numtc++;
1588                 }
1589                 if (!numtc) {
1590                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1591                         numtc = 1;
1592                 }
1593         } else {
1594                 /* At least TC0 is enabled in case of non-DCB case */
1595                 numtc = 1;
1596         }
1597
1598         vsi->tc_config.numtc = numtc;
1599         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1600         /* Number of queues per enabled TC */
1601         qcount = vsi->alloc_queue_pairs;
1602
1603         num_tc_qps = qcount / numtc;
1604         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1605
1606         /* Setup queue offset/count for all TCs for given VSI */
1607         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1608                 /* See if the given TC is enabled for the given VSI */
1609                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1610                         /* TC is enabled */
1611                         int pow, num_qps;
1612
1613                         switch (vsi->type) {
1614                         case I40E_VSI_MAIN:
1615                                 qcount = min_t(int, pf->alloc_rss_size,
1616                                                num_tc_qps);
1617                                 break;
1618                         case I40E_VSI_FDIR:
1619                         case I40E_VSI_SRIOV:
1620                         case I40E_VSI_VMDQ2:
1621                         default:
1622                                 qcount = num_tc_qps;
1623                                 WARN_ON(i != 0);
1624                                 break;
1625                         }
1626                         vsi->tc_config.tc_info[i].qoffset = offset;
1627                         vsi->tc_config.tc_info[i].qcount = qcount;
1628
1629                         /* find the next higher power-of-2 of num queue pairs */
1630                         num_qps = qcount;
1631                         pow = 0;
1632                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1633                                 pow++;
1634                                 num_qps >>= 1;
1635                         }
1636
1637                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1638                         qmap =
1639                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1640                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1641
1642                         offset += qcount;
1643                 } else {
1644                         /* TC is not enabled so set the offset to
1645                          * default queue and allocate one queue
1646                          * for the given TC.
1647                          */
1648                         vsi->tc_config.tc_info[i].qoffset = 0;
1649                         vsi->tc_config.tc_info[i].qcount = 1;
1650                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1651
1652                         qmap = 0;
1653                 }
1654                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1655         }
1656
1657         /* Set actual Tx/Rx queue pairs */
1658         vsi->num_queue_pairs = offset;
1659         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1660                 if (vsi->req_queue_pairs > 0)
1661                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1662                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1663                         vsi->num_queue_pairs = pf->num_lan_msix;
1664         }
1665
1666         /* Scheduler section valid can only be set for ADD VSI */
1667         if (is_add) {
1668                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1669
1670                 ctxt->info.up_enable_bits = enabled_tc;
1671         }
1672         if (vsi->type == I40E_VSI_SRIOV) {
1673                 ctxt->info.mapping_flags |=
1674                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1675                 for (i = 0; i < vsi->num_queue_pairs; i++)
1676                         ctxt->info.queue_mapping[i] =
1677                                                cpu_to_le16(vsi->base_queue + i);
1678         } else {
1679                 ctxt->info.mapping_flags |=
1680                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1681                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1682         }
1683         ctxt->info.valid_sections |= cpu_to_le16(sections);
1684 }
1685
1686 /**
1687  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1688  * @netdev: the netdevice
1689  * @addr: address to add
1690  *
1691  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1692  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1693  */
1694 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1695 {
1696         struct i40e_netdev_priv *np = netdev_priv(netdev);
1697         struct i40e_vsi *vsi = np->vsi;
1698
1699         if (i40e_add_mac_filter(vsi, addr))
1700                 return 0;
1701         else
1702                 return -ENOMEM;
1703 }
1704
1705 /**
1706  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1707  * @netdev: the netdevice
1708  * @addr: address to add
1709  *
1710  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1711  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1712  */
1713 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1714 {
1715         struct i40e_netdev_priv *np = netdev_priv(netdev);
1716         struct i40e_vsi *vsi = np->vsi;
1717
1718         i40e_del_mac_filter(vsi, addr);
1719
1720         return 0;
1721 }
1722
1723 /**
1724  * i40e_set_rx_mode - NDO callback to set the netdev filters
1725  * @netdev: network interface device structure
1726  **/
1727 static void i40e_set_rx_mode(struct net_device *netdev)
1728 {
1729         struct i40e_netdev_priv *np = netdev_priv(netdev);
1730         struct i40e_vsi *vsi = np->vsi;
1731
1732         spin_lock_bh(&vsi->mac_filter_hash_lock);
1733
1734         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1735         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1736
1737         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1738
1739         /* check for other flag changes */
1740         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1741                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1742                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1743         }
1744
1745         /* schedule our worker thread which will take care of
1746          * applying the new filter changes
1747          */
1748         i40e_service_event_schedule(vsi->back);
1749 }
1750
1751 /**
1752  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
1753  * @vsi: Pointer to VSI struct
1754  * @from: Pointer to list which contains MAC filter entries - changes to
1755  *        those entries needs to be undone.
1756  *
1757  * MAC filter entries from this list were slated for deletion.
1758  **/
1759 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1760                                          struct hlist_head *from)
1761 {
1762         struct i40e_mac_filter *f;
1763         struct hlist_node *h;
1764
1765         hlist_for_each_entry_safe(f, h, from, hlist) {
1766                 u64 key = i40e_addr_to_hkey(f->macaddr);
1767
1768                 /* Move the element back into MAC filter list*/
1769                 hlist_del(&f->hlist);
1770                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1771         }
1772 }
1773
1774 /**
1775  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
1776  * @vsi: Pointer to vsi struct
1777  * @from: Pointer to list which contains MAC filter entries - changes to
1778  *        those entries needs to be undone.
1779  *
1780  * MAC filter entries from this list were slated for addition.
1781  **/
1782 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1783                                          struct hlist_head *from)
1784 {
1785         struct i40e_new_mac_filter *new;
1786         struct hlist_node *h;
1787
1788         hlist_for_each_entry_safe(new, h, from, hlist) {
1789                 /* We can simply free the wrapper structure */
1790                 hlist_del(&new->hlist);
1791                 kfree(new);
1792         }
1793 }
1794
1795 /**
1796  * i40e_next_entry - Get the next non-broadcast filter from a list
1797  * @next: pointer to filter in list
1798  *
1799  * Returns the next non-broadcast filter in the list. Required so that we
1800  * ignore broadcast filters within the list, since these are not handled via
1801  * the normal firmware update path.
1802  */
1803 static
1804 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
1805 {
1806         hlist_for_each_entry_continue(next, hlist) {
1807                 if (!is_broadcast_ether_addr(next->f->macaddr))
1808                         return next;
1809         }
1810
1811         return NULL;
1812 }
1813
1814 /**
1815  * i40e_update_filter_state - Update filter state based on return data
1816  * from firmware
1817  * @count: Number of filters added
1818  * @add_list: return data from fw
1819  * @head: pointer to first filter in current batch
1820  *
1821  * MAC filter entries from list were slated to be added to device. Returns
1822  * number of successful filters. Note that 0 does NOT mean success!
1823  **/
1824 static int
1825 i40e_update_filter_state(int count,
1826                          struct i40e_aqc_add_macvlan_element_data *add_list,
1827                          struct i40e_new_mac_filter *add_head)
1828 {
1829         int retval = 0;
1830         int i;
1831
1832         for (i = 0; i < count; i++) {
1833                 /* Always check status of each filter. We don't need to check
1834                  * the firmware return status because we pre-set the filter
1835                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1836                  * request to the adminq. Thus, if it no longer matches then
1837                  * we know the filter is active.
1838                  */
1839                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1840                         add_head->state = I40E_FILTER_FAILED;
1841                 } else {
1842                         add_head->state = I40E_FILTER_ACTIVE;
1843                         retval++;
1844                 }
1845
1846                 add_head = i40e_next_filter(add_head);
1847                 if (!add_head)
1848                         break;
1849         }
1850
1851         return retval;
1852 }
1853
1854 /**
1855  * i40e_aqc_del_filters - Request firmware to delete a set of filters
1856  * @vsi: ptr to the VSI
1857  * @vsi_name: name to display in messages
1858  * @list: the list of filters to send to firmware
1859  * @num_del: the number of filters to delete
1860  * @retval: Set to -EIO on failure to delete
1861  *
1862  * Send a request to firmware via AdminQ to delete a set of filters. Uses
1863  * *retval instead of a return value so that success does not force ret_val to
1864  * be set to 0. This ensures that a sequence of calls to this function
1865  * preserve the previous value of *retval on successful delete.
1866  */
1867 static
1868 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1869                           struct i40e_aqc_remove_macvlan_element_data *list,
1870                           int num_del, int *retval)
1871 {
1872         struct i40e_hw *hw = &vsi->back->hw;
1873         i40e_status aq_ret;
1874         int aq_err;
1875
1876         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1877         aq_err = hw->aq.asq_last_status;
1878
1879         /* Explicitly ignore and do not report when firmware returns ENOENT */
1880         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1881                 *retval = -EIO;
1882                 dev_info(&vsi->back->pdev->dev,
1883                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1884                          vsi_name, i40e_stat_str(hw, aq_ret),
1885                          i40e_aq_str(hw, aq_err));
1886         }
1887 }
1888
1889 /**
1890  * i40e_aqc_add_filters - Request firmware to add a set of filters
1891  * @vsi: ptr to the VSI
1892  * @vsi_name: name to display in messages
1893  * @list: the list of filters to send to firmware
1894  * @add_head: Position in the add hlist
1895  * @num_add: the number of filters to add
1896  * @promisc_change: set to true on exit if promiscuous mode was forced on
1897  *
1898  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1899  * promisc_changed to true if the firmware has run out of space for more
1900  * filters.
1901  */
1902 static
1903 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1904                           struct i40e_aqc_add_macvlan_element_data *list,
1905                           struct i40e_new_mac_filter *add_head,
1906                           int num_add, bool *promisc_changed)
1907 {
1908         struct i40e_hw *hw = &vsi->back->hw;
1909         int aq_err, fcnt;
1910
1911         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1912         aq_err = hw->aq.asq_last_status;
1913         fcnt = i40e_update_filter_state(num_add, list, add_head);
1914
1915         if (fcnt != num_add) {
1916                 *promisc_changed = true;
1917                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
1918                 dev_warn(&vsi->back->pdev->dev,
1919                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1920                          i40e_aq_str(hw, aq_err),
1921                          vsi_name);
1922         }
1923 }
1924
1925 /**
1926  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1927  * @vsi: pointer to the VSI
1928  * @f: filter data
1929  *
1930  * This function sets or clears the promiscuous broadcast flags for VLAN
1931  * filters in order to properly receive broadcast frames. Assumes that only
1932  * broadcast filters are passed.
1933  *
1934  * Returns status indicating success or failure;
1935  **/
1936 static i40e_status
1937 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1938                           struct i40e_mac_filter *f)
1939 {
1940         bool enable = f->state == I40E_FILTER_NEW;
1941         struct i40e_hw *hw = &vsi->back->hw;
1942         i40e_status aq_ret;
1943
1944         if (f->vlan == I40E_VLAN_ANY) {
1945                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1946                                                    vsi->seid,
1947                                                    enable,
1948                                                    NULL);
1949         } else {
1950                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1951                                                             vsi->seid,
1952                                                             enable,
1953                                                             f->vlan,
1954                                                             NULL);
1955         }
1956
1957         if (aq_ret)
1958                 dev_warn(&vsi->back->pdev->dev,
1959                          "Error %s setting broadcast promiscuous mode on %s\n",
1960                          i40e_aq_str(hw, hw->aq.asq_last_status),
1961                          vsi_name);
1962
1963         return aq_ret;
1964 }
1965
1966 /**
1967  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
1968  * @vsi: ptr to the VSI
1969  *
1970  * Push any outstanding VSI filter changes through the AdminQ.
1971  *
1972  * Returns 0 or error value
1973  **/
1974 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
1975 {
1976         struct hlist_head tmp_add_list, tmp_del_list;
1977         struct i40e_mac_filter *f;
1978         struct i40e_new_mac_filter *new, *add_head = NULL;
1979         struct i40e_hw *hw = &vsi->back->hw;
1980         unsigned int failed_filters = 0;
1981         unsigned int vlan_filters = 0;
1982         bool promisc_changed = false;
1983         char vsi_name[16] = "PF";
1984         int filter_list_len = 0;
1985         i40e_status aq_ret = 0;
1986         u32 changed_flags = 0;
1987         struct hlist_node *h;
1988         struct i40e_pf *pf;
1989         int num_add = 0;
1990         int num_del = 0;
1991         int retval = 0;
1992         u16 cmd_flags;
1993         int list_size;
1994         int bkt;
1995
1996         /* empty array typed pointers, kcalloc later */
1997         struct i40e_aqc_add_macvlan_element_data *add_list;
1998         struct i40e_aqc_remove_macvlan_element_data *del_list;
1999
2000         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
2001                 usleep_range(1000, 2000);
2002         pf = vsi->back;
2003
2004         if (vsi->netdev) {
2005                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2006                 vsi->current_netdev_flags = vsi->netdev->flags;
2007         }
2008
2009         INIT_HLIST_HEAD(&tmp_add_list);
2010         INIT_HLIST_HEAD(&tmp_del_list);
2011
2012         if (vsi->type == I40E_VSI_SRIOV)
2013                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2014         else if (vsi->type != I40E_VSI_MAIN)
2015                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2016
2017         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2018                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2019
2020                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2021                 /* Create a list of filters to delete. */
2022                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2023                         if (f->state == I40E_FILTER_REMOVE) {
2024                                 /* Move the element into temporary del_list */
2025                                 hash_del(&f->hlist);
2026                                 hlist_add_head(&f->hlist, &tmp_del_list);
2027
2028                                 /* Avoid counting removed filters */
2029                                 continue;
2030                         }
2031                         if (f->state == I40E_FILTER_NEW) {
2032                                 /* Create a temporary i40e_new_mac_filter */
2033                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2034                                 if (!new)
2035                                         goto err_no_memory_locked;
2036
2037                                 /* Store pointer to the real filter */
2038                                 new->f = f;
2039                                 new->state = f->state;
2040
2041                                 /* Add it to the hash list */
2042                                 hlist_add_head(&new->hlist, &tmp_add_list);
2043                         }
2044
2045                         /* Count the number of active (current and new) VLAN
2046                          * filters we have now. Does not count filters which
2047                          * are marked for deletion.
2048                          */
2049                         if (f->vlan > 0)
2050                                 vlan_filters++;
2051                 }
2052
2053                 retval = i40e_correct_mac_vlan_filters(vsi,
2054                                                        &tmp_add_list,
2055                                                        &tmp_del_list,
2056                                                        vlan_filters);
2057                 if (retval)
2058                         goto err_no_memory_locked;
2059
2060                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2061         }
2062
2063         /* Now process 'del_list' outside the lock */
2064         if (!hlist_empty(&tmp_del_list)) {
2065                 filter_list_len = hw->aq.asq_buf_size /
2066                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2067                 list_size = filter_list_len *
2068                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2069                 del_list = kzalloc(list_size, GFP_ATOMIC);
2070                 if (!del_list)
2071                         goto err_no_memory;
2072
2073                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2074                         cmd_flags = 0;
2075
2076                         /* handle broadcast filters by updating the broadcast
2077                          * promiscuous flag and release filter list.
2078                          */
2079                         if (is_broadcast_ether_addr(f->macaddr)) {
2080                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2081
2082                                 hlist_del(&f->hlist);
2083                                 kfree(f);
2084                                 continue;
2085                         }
2086
2087                         /* add to delete list */
2088                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2089                         if (f->vlan == I40E_VLAN_ANY) {
2090                                 del_list[num_del].vlan_tag = 0;
2091                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2092                         } else {
2093                                 del_list[num_del].vlan_tag =
2094                                         cpu_to_le16((u16)(f->vlan));
2095                         }
2096
2097                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2098                         del_list[num_del].flags = cmd_flags;
2099                         num_del++;
2100
2101                         /* flush a full buffer */
2102                         if (num_del == filter_list_len) {
2103                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2104                                                      num_del, &retval);
2105                                 memset(del_list, 0, list_size);
2106                                 num_del = 0;
2107                         }
2108                         /* Release memory for MAC filter entries which were
2109                          * synced up with HW.
2110                          */
2111                         hlist_del(&f->hlist);
2112                         kfree(f);
2113                 }
2114
2115                 if (num_del) {
2116                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2117                                              num_del, &retval);
2118                 }
2119
2120                 kfree(del_list);
2121                 del_list = NULL;
2122         }
2123
2124         if (!hlist_empty(&tmp_add_list)) {
2125                 /* Do all the adds now. */
2126                 filter_list_len = hw->aq.asq_buf_size /
2127                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2128                 list_size = filter_list_len *
2129                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2130                 add_list = kzalloc(list_size, GFP_ATOMIC);
2131                 if (!add_list)
2132                         goto err_no_memory;
2133
2134                 num_add = 0;
2135                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2136                         if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2137                                      &vsi->state)) {
2138                                 new->state = I40E_FILTER_FAILED;
2139                                 continue;
2140                         }
2141
2142                         /* handle broadcast filters by updating the broadcast
2143                          * promiscuous flag instead of adding a MAC filter.
2144                          */
2145                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2146                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2147                                                               new->f))
2148                                         new->state = I40E_FILTER_FAILED;
2149                                 else
2150                                         new->state = I40E_FILTER_ACTIVE;
2151                                 continue;
2152                         }
2153
2154                         /* add to add array */
2155                         if (num_add == 0)
2156                                 add_head = new;
2157                         cmd_flags = 0;
2158                         ether_addr_copy(add_list[num_add].mac_addr,
2159                                         new->f->macaddr);
2160                         if (new->f->vlan == I40E_VLAN_ANY) {
2161                                 add_list[num_add].vlan_tag = 0;
2162                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2163                         } else {
2164                                 add_list[num_add].vlan_tag =
2165                                         cpu_to_le16((u16)(new->f->vlan));
2166                         }
2167                         add_list[num_add].queue_number = 0;
2168                         /* set invalid match method for later detection */
2169                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2170                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2171                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2172                         num_add++;
2173
2174                         /* flush a full buffer */
2175                         if (num_add == filter_list_len) {
2176                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2177                                                      add_head, num_add,
2178                                                      &promisc_changed);
2179                                 memset(add_list, 0, list_size);
2180                                 num_add = 0;
2181                         }
2182                 }
2183                 if (num_add) {
2184                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2185                                              num_add, &promisc_changed);
2186                 }
2187                 /* Now move all of the filters from the temp add list back to
2188                  * the VSI's list.
2189                  */
2190                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2191                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2192                         /* Only update the state if we're still NEW */
2193                         if (new->f->state == I40E_FILTER_NEW)
2194                                 new->f->state = new->state;
2195                         hlist_del(&new->hlist);
2196                         kfree(new);
2197                 }
2198                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2199                 kfree(add_list);
2200                 add_list = NULL;
2201         }
2202
2203         /* Determine the number of active and failed filters. */
2204         spin_lock_bh(&vsi->mac_filter_hash_lock);
2205         vsi->active_filters = 0;
2206         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2207                 if (f->state == I40E_FILTER_ACTIVE)
2208                         vsi->active_filters++;
2209                 else if (f->state == I40E_FILTER_FAILED)
2210                         failed_filters++;
2211         }
2212         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2213
2214         /* If promiscuous mode has changed, we need to calculate a new
2215          * threshold for when we are safe to exit
2216          */
2217         if (promisc_changed)
2218                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2219
2220         /* Check if we are able to exit overflow promiscuous mode. We can
2221          * safely exit if we didn't just enter, we no longer have any failed
2222          * filters, and we have reduced filters below the threshold value.
2223          */
2224         if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state) &&
2225             !promisc_changed && !failed_filters &&
2226             (vsi->active_filters < vsi->promisc_threshold)) {
2227                 dev_info(&pf->pdev->dev,
2228                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2229                          vsi_name);
2230                 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
2231                 promisc_changed = true;
2232                 vsi->promisc_threshold = 0;
2233         }
2234
2235         /* if the VF is not trusted do not do promisc */
2236         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2237                 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
2238                 goto out;
2239         }
2240
2241         /* check for changes in promiscuous modes */
2242         if (changed_flags & IFF_ALLMULTI) {
2243                 bool cur_multipromisc;
2244
2245                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2246                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2247                                                                vsi->seid,
2248                                                                cur_multipromisc,
2249                                                                NULL);
2250                 if (aq_ret) {
2251                         retval = i40e_aq_rc_to_posix(aq_ret,
2252                                                      hw->aq.asq_last_status);
2253                         dev_info(&pf->pdev->dev,
2254                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2255                                  vsi_name,
2256                                  i40e_stat_str(hw, aq_ret),
2257                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2258                 }
2259         }
2260         if ((changed_flags & IFF_PROMISC) ||
2261             (promisc_changed &&
2262              test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))) {
2263                 bool cur_promisc;
2264
2265                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2266                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2267                                         &vsi->state));
2268                 if ((vsi->type == I40E_VSI_MAIN) &&
2269                     (pf->lan_veb != I40E_NO_VEB) &&
2270                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2271                         /* set defport ON for Main VSI instead of true promisc
2272                          * this way we will get all unicast/multicast and VLAN
2273                          * promisc behavior but will not get VF or VMDq traffic
2274                          * replicated on the Main VSI.
2275                          */
2276                         if (pf->cur_promisc != cur_promisc) {
2277                                 pf->cur_promisc = cur_promisc;
2278                                 if (cur_promisc)
2279                                         aq_ret =
2280                                               i40e_aq_set_default_vsi(hw,
2281                                                                       vsi->seid,
2282                                                                       NULL);
2283                                 else
2284                                         aq_ret =
2285                                             i40e_aq_clear_default_vsi(hw,
2286                                                                       vsi->seid,
2287                                                                       NULL);
2288                                 if (aq_ret) {
2289                                         retval = i40e_aq_rc_to_posix(aq_ret,
2290                                                         hw->aq.asq_last_status);
2291                                         dev_info(&pf->pdev->dev,
2292                                                  "Set default VSI failed on %s, err %s, aq_err %s\n",
2293                                                  vsi_name,
2294                                                  i40e_stat_str(hw, aq_ret),
2295                                                  i40e_aq_str(hw,
2296                                                      hw->aq.asq_last_status));
2297                                 }
2298                         }
2299                 } else {
2300                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2301                                                           hw,
2302                                                           vsi->seid,
2303                                                           cur_promisc, NULL,
2304                                                           true);
2305                         if (aq_ret) {
2306                                 retval =
2307                                 i40e_aq_rc_to_posix(aq_ret,
2308                                                     hw->aq.asq_last_status);
2309                                 dev_info(&pf->pdev->dev,
2310                                          "set unicast promisc failed on %s, err %s, aq_err %s\n",
2311                                          vsi_name,
2312                                          i40e_stat_str(hw, aq_ret),
2313                                          i40e_aq_str(hw,
2314                                                      hw->aq.asq_last_status));
2315                         }
2316                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2317                                                           hw,
2318                                                           vsi->seid,
2319                                                           cur_promisc, NULL);
2320                         if (aq_ret) {
2321                                 retval =
2322                                 i40e_aq_rc_to_posix(aq_ret,
2323                                                     hw->aq.asq_last_status);
2324                                 dev_info(&pf->pdev->dev,
2325                                          "set multicast promisc failed on %s, err %s, aq_err %s\n",
2326                                          vsi_name,
2327                                          i40e_stat_str(hw, aq_ret),
2328                                          i40e_aq_str(hw,
2329                                                      hw->aq.asq_last_status));
2330                         }
2331                 }
2332                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2333                                                    vsi->seid,
2334                                                    cur_promisc, NULL);
2335                 if (aq_ret) {
2336                         retval = i40e_aq_rc_to_posix(aq_ret,
2337                                                      pf->hw.aq.asq_last_status);
2338                         dev_info(&pf->pdev->dev,
2339                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2340                                          i40e_stat_str(hw, aq_ret),
2341                                          i40e_aq_str(hw,
2342                                                      hw->aq.asq_last_status));
2343                 }
2344         }
2345 out:
2346         /* if something went wrong then set the changed flag so we try again */
2347         if (retval)
2348                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2349
2350         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
2351         return retval;
2352
2353 err_no_memory:
2354         /* Restore elements on the temporary add and delete lists */
2355         spin_lock_bh(&vsi->mac_filter_hash_lock);
2356 err_no_memory_locked:
2357         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2358         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2359         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2360
2361         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2362         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
2363         return -ENOMEM;
2364 }
2365
2366 /**
2367  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2368  * @pf: board private structure
2369  **/
2370 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2371 {
2372         int v;
2373
2374         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2375                 return;
2376         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2377
2378         for (v = 0; v < pf->num_alloc_vsi; v++) {
2379                 if (pf->vsi[v] &&
2380                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2381                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2382
2383                         if (ret) {
2384                                 /* come back and try again later */
2385                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2386                                 break;
2387                         }
2388                 }
2389         }
2390 }
2391
2392 /**
2393  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2394  * @netdev: network interface device structure
2395  * @new_mtu: new value for maximum frame size
2396  *
2397  * Returns 0 on success, negative on failure
2398  **/
2399 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2400 {
2401         struct i40e_netdev_priv *np = netdev_priv(netdev);
2402         struct i40e_vsi *vsi = np->vsi;
2403         struct i40e_pf *pf = vsi->back;
2404
2405         netdev_info(netdev, "changing MTU from %d to %d\n",
2406                     netdev->mtu, new_mtu);
2407         netdev->mtu = new_mtu;
2408         if (netif_running(netdev))
2409                 i40e_vsi_reinit_locked(vsi);
2410         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
2411                       I40E_FLAG_CLIENT_L2_CHANGE);
2412         return 0;
2413 }
2414
2415 /**
2416  * i40e_ioctl - Access the hwtstamp interface
2417  * @netdev: network interface device structure
2418  * @ifr: interface request data
2419  * @cmd: ioctl command
2420  **/
2421 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2422 {
2423         struct i40e_netdev_priv *np = netdev_priv(netdev);
2424         struct i40e_pf *pf = np->vsi->back;
2425
2426         switch (cmd) {
2427         case SIOCGHWTSTAMP:
2428                 return i40e_ptp_get_ts_config(pf, ifr);
2429         case SIOCSHWTSTAMP:
2430                 return i40e_ptp_set_ts_config(pf, ifr);
2431         default:
2432                 return -EOPNOTSUPP;
2433         }
2434 }
2435
2436 /**
2437  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2438  * @vsi: the vsi being adjusted
2439  **/
2440 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2441 {
2442         struct i40e_vsi_context ctxt;
2443         i40e_status ret;
2444
2445         if ((vsi->info.valid_sections &
2446              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2447             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2448                 return;  /* already enabled */
2449
2450         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2451         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2452                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2453
2454         ctxt.seid = vsi->seid;
2455         ctxt.info = vsi->info;
2456         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2457         if (ret) {
2458                 dev_info(&vsi->back->pdev->dev,
2459                          "update vlan stripping failed, err %s aq_err %s\n",
2460                          i40e_stat_str(&vsi->back->hw, ret),
2461                          i40e_aq_str(&vsi->back->hw,
2462                                      vsi->back->hw.aq.asq_last_status));
2463         }
2464 }
2465
2466 /**
2467  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2468  * @vsi: the vsi being adjusted
2469  **/
2470 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2471 {
2472         struct i40e_vsi_context ctxt;
2473         i40e_status ret;
2474
2475         if ((vsi->info.valid_sections &
2476              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2477             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2478              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2479                 return;  /* already disabled */
2480
2481         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2482         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2483                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2484
2485         ctxt.seid = vsi->seid;
2486         ctxt.info = vsi->info;
2487         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2488         if (ret) {
2489                 dev_info(&vsi->back->pdev->dev,
2490                          "update vlan stripping failed, err %s aq_err %s\n",
2491                          i40e_stat_str(&vsi->back->hw, ret),
2492                          i40e_aq_str(&vsi->back->hw,
2493                                      vsi->back->hw.aq.asq_last_status));
2494         }
2495 }
2496
2497 /**
2498  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2499  * @netdev: network interface to be adjusted
2500  * @features: netdev features to test if VLAN offload is enabled or not
2501  **/
2502 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2503 {
2504         struct i40e_netdev_priv *np = netdev_priv(netdev);
2505         struct i40e_vsi *vsi = np->vsi;
2506
2507         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2508                 i40e_vlan_stripping_enable(vsi);
2509         else
2510                 i40e_vlan_stripping_disable(vsi);
2511 }
2512
2513 /**
2514  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2515  * @vsi: the vsi being configured
2516  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2517  *
2518  * This is a helper function for adding a new MAC/VLAN filter with the
2519  * specified VLAN for each existing MAC address already in the hash table.
2520  * This function does *not* perform any accounting to update filters based on
2521  * VLAN mode.
2522  *
2523  * NOTE: this function expects to be called while under the
2524  * mac_filter_hash_lock
2525  **/
2526 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2527 {
2528         struct i40e_mac_filter *f, *add_f;
2529         struct hlist_node *h;
2530         int bkt;
2531
2532         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2533                 if (f->state == I40E_FILTER_REMOVE)
2534                         continue;
2535                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2536                 if (!add_f) {
2537                         dev_info(&vsi->back->pdev->dev,
2538                                  "Could not add vlan filter %d for %pM\n",
2539                                  vid, f->macaddr);
2540                         return -ENOMEM;
2541                 }
2542         }
2543
2544         return 0;
2545 }
2546
2547 /**
2548  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2549  * @vsi: the VSI being configured
2550  * @vid: VLAN id to be added
2551  **/
2552 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2553 {
2554         int err;
2555
2556         if (!vid || vsi->info.pvid)
2557                 return -EINVAL;
2558
2559         /* Locked once because all functions invoked below iterates list*/
2560         spin_lock_bh(&vsi->mac_filter_hash_lock);
2561         err = i40e_add_vlan_all_mac(vsi, vid);
2562         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2563         if (err)
2564                 return err;
2565
2566         /* schedule our worker thread which will take care of
2567          * applying the new filter changes
2568          */
2569         i40e_service_event_schedule(vsi->back);
2570         return 0;
2571 }
2572
2573 /**
2574  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2575  * @vsi: the vsi being configured
2576  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2577  *
2578  * This function should be used to remove all VLAN filters which match the
2579  * given VID. It does not schedule the service event and does not take the
2580  * mac_filter_hash_lock so it may be combined with other operations under
2581  * a single invocation of the mac_filter_hash_lock.
2582  *
2583  * NOTE: this function expects to be called while under the
2584  * mac_filter_hash_lock
2585  */
2586 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2587 {
2588         struct i40e_mac_filter *f;
2589         struct hlist_node *h;
2590         int bkt;
2591
2592         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2593                 if (f->vlan == vid)
2594                         __i40e_del_filter(vsi, f);
2595         }
2596 }
2597
2598 /**
2599  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2600  * @vsi: the VSI being configured
2601  * @vid: VLAN id to be removed
2602  **/
2603 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2604 {
2605         if (!vid || vsi->info.pvid)
2606                 return;
2607
2608         spin_lock_bh(&vsi->mac_filter_hash_lock);
2609         i40e_rm_vlan_all_mac(vsi, vid);
2610         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2611
2612         /* schedule our worker thread which will take care of
2613          * applying the new filter changes
2614          */
2615         i40e_service_event_schedule(vsi->back);
2616 }
2617
2618 /**
2619  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2620  * @netdev: network interface to be adjusted
2621  * @vid: vlan id to be added
2622  *
2623  * net_device_ops implementation for adding vlan ids
2624  **/
2625 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2626                                 __always_unused __be16 proto, u16 vid)
2627 {
2628         struct i40e_netdev_priv *np = netdev_priv(netdev);
2629         struct i40e_vsi *vsi = np->vsi;
2630         int ret = 0;
2631
2632         if (vid >= VLAN_N_VID)
2633                 return -EINVAL;
2634
2635         /* If the network stack called us with vid = 0 then
2636          * it is asking to receive priority tagged packets with
2637          * vlan id 0.  Our HW receives them by default when configured
2638          * to receive untagged packets so there is no need to add an
2639          * extra filter for vlan 0 tagged packets.
2640          */
2641         if (vid)
2642                 ret = i40e_vsi_add_vlan(vsi, vid);
2643
2644         if (!ret)
2645                 set_bit(vid, vsi->active_vlans);
2646
2647         return ret;
2648 }
2649
2650 /**
2651  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2652  * @netdev: network interface to be adjusted
2653  * @vid: vlan id to be removed
2654  *
2655  * net_device_ops implementation for removing vlan ids
2656  **/
2657 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2658                                  __always_unused __be16 proto, u16 vid)
2659 {
2660         struct i40e_netdev_priv *np = netdev_priv(netdev);
2661         struct i40e_vsi *vsi = np->vsi;
2662
2663         /* return code is ignored as there is nothing a user
2664          * can do about failure to remove and a log message was
2665          * already printed from the other function
2666          */
2667         i40e_vsi_kill_vlan(vsi, vid);
2668
2669         clear_bit(vid, vsi->active_vlans);
2670
2671         return 0;
2672 }
2673
2674 /**
2675  * i40e_macaddr_init - explicitly write the mac address filters
2676  *
2677  * @vsi: pointer to the vsi
2678  * @macaddr: the MAC address
2679  *
2680  * This is needed when the macaddr has been obtained by other
2681  * means than the default, e.g., from Open Firmware or IDPROM.
2682  * Returns 0 on success, negative on failure
2683  **/
2684 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr)
2685 {
2686         int ret;
2687         struct i40e_aqc_add_macvlan_element_data element;
2688
2689         ret = i40e_aq_mac_address_write(&vsi->back->hw,
2690                                         I40E_AQC_WRITE_TYPE_LAA_WOL,
2691                                         macaddr, NULL);
2692         if (ret) {
2693                 dev_info(&vsi->back->pdev->dev,
2694                          "Addr change for VSI failed: %d\n", ret);
2695                 return -EADDRNOTAVAIL;
2696         }
2697
2698         memset(&element, 0, sizeof(element));
2699         ether_addr_copy(element.mac_addr, macaddr);
2700         element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
2701         ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL);
2702         if (ret) {
2703                 dev_info(&vsi->back->pdev->dev,
2704                          "add filter failed err %s aq_err %s\n",
2705                          i40e_stat_str(&vsi->back->hw, ret),
2706                          i40e_aq_str(&vsi->back->hw,
2707                                      vsi->back->hw.aq.asq_last_status));
2708         }
2709         return ret;
2710 }
2711
2712 /**
2713  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2714  * @vsi: the vsi being brought back up
2715  **/
2716 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2717 {
2718         u16 vid;
2719
2720         if (!vsi->netdev)
2721                 return;
2722
2723         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2724
2725         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2726                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2727                                      vid);
2728 }
2729
2730 /**
2731  * i40e_vsi_add_pvid - Add pvid for the VSI
2732  * @vsi: the vsi being adjusted
2733  * @vid: the vlan id to set as a PVID
2734  **/
2735 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2736 {
2737         struct i40e_vsi_context ctxt;
2738         i40e_status ret;
2739
2740         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2741         vsi->info.pvid = cpu_to_le16(vid);
2742         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2743                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2744                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2745
2746         ctxt.seid = vsi->seid;
2747         ctxt.info = vsi->info;
2748         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2749         if (ret) {
2750                 dev_info(&vsi->back->pdev->dev,
2751                          "add pvid failed, err %s aq_err %s\n",
2752                          i40e_stat_str(&vsi->back->hw, ret),
2753                          i40e_aq_str(&vsi->back->hw,
2754                                      vsi->back->hw.aq.asq_last_status));
2755                 return -ENOENT;
2756         }
2757
2758         return 0;
2759 }
2760
2761 /**
2762  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2763  * @vsi: the vsi being adjusted
2764  *
2765  * Just use the vlan_rx_register() service to put it back to normal
2766  **/
2767 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2768 {
2769         i40e_vlan_stripping_disable(vsi);
2770
2771         vsi->info.pvid = 0;
2772 }
2773
2774 /**
2775  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2776  * @vsi: ptr to the VSI
2777  *
2778  * If this function returns with an error, then it's possible one or
2779  * more of the rings is populated (while the rest are not).  It is the
2780  * callers duty to clean those orphaned rings.
2781  *
2782  * Return 0 on success, negative on failure
2783  **/
2784 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2785 {
2786         int i, err = 0;
2787
2788         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2789                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2790
2791         return err;
2792 }
2793
2794 /**
2795  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2796  * @vsi: ptr to the VSI
2797  *
2798  * Free VSI's transmit software resources
2799  **/
2800 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2801 {
2802         int i;
2803
2804         if (!vsi->tx_rings)
2805                 return;
2806
2807         for (i = 0; i < vsi->num_queue_pairs; i++)
2808                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2809                         i40e_free_tx_resources(vsi->tx_rings[i]);
2810 }
2811
2812 /**
2813  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2814  * @vsi: ptr to the VSI
2815  *
2816  * If this function returns with an error, then it's possible one or
2817  * more of the rings is populated (while the rest are not).  It is the
2818  * callers duty to clean those orphaned rings.
2819  *
2820  * Return 0 on success, negative on failure
2821  **/
2822 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2823 {
2824         int i, err = 0;
2825
2826         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2827                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2828         return err;
2829 }
2830
2831 /**
2832  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2833  * @vsi: ptr to the VSI
2834  *
2835  * Free all receive software resources
2836  **/
2837 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2838 {
2839         int i;
2840
2841         if (!vsi->rx_rings)
2842                 return;
2843
2844         for (i = 0; i < vsi->num_queue_pairs; i++)
2845                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2846                         i40e_free_rx_resources(vsi->rx_rings[i]);
2847 }
2848
2849 /**
2850  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2851  * @ring: The Tx ring to configure
2852  *
2853  * This enables/disables XPS for a given Tx descriptor ring
2854  * based on the TCs enabled for the VSI that ring belongs to.
2855  **/
2856 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2857 {
2858         struct i40e_vsi *vsi = ring->vsi;
2859         cpumask_var_t mask;
2860
2861         if (!ring->q_vector || !ring->netdev)
2862                 return;
2863
2864         /* Single TC mode enable XPS */
2865         if (vsi->tc_config.numtc <= 1) {
2866                 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2867                         netif_set_xps_queue(ring->netdev,
2868                                             &ring->q_vector->affinity_mask,
2869                                             ring->queue_index);
2870         } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
2871                 /* Disable XPS to allow selection based on TC */
2872                 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits);
2873                 netif_set_xps_queue(ring->netdev, mask, ring->queue_index);
2874                 free_cpumask_var(mask);
2875         }
2876
2877         /* schedule our worker thread which will take care of
2878          * applying the new filter changes
2879          */
2880         i40e_service_event_schedule(vsi->back);
2881 }
2882
2883 /**
2884  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2885  * @ring: The Tx ring to configure
2886  *
2887  * Configure the Tx descriptor ring in the HMC context.
2888  **/
2889 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2890 {
2891         struct i40e_vsi *vsi = ring->vsi;
2892         u16 pf_q = vsi->base_queue + ring->queue_index;
2893         struct i40e_hw *hw = &vsi->back->hw;
2894         struct i40e_hmc_obj_txq tx_ctx;
2895         i40e_status err = 0;
2896         u32 qtx_ctl = 0;
2897
2898         /* some ATR related tx ring init */
2899         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2900                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2901                 ring->atr_count = 0;
2902         } else {
2903                 ring->atr_sample_rate = 0;
2904         }
2905
2906         /* configure XPS */
2907         i40e_config_xps_tx_ring(ring);
2908
2909         /* clear the context structure first */
2910         memset(&tx_ctx, 0, sizeof(tx_ctx));
2911
2912         tx_ctx.new_context = 1;
2913         tx_ctx.base = (ring->dma / 128);
2914         tx_ctx.qlen = ring->count;
2915         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2916                                                I40E_FLAG_FD_ATR_ENABLED));
2917         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2918         /* FDIR VSI tx ring can still use RS bit and writebacks */
2919         if (vsi->type != I40E_VSI_FDIR)
2920                 tx_ctx.head_wb_ena = 1;
2921         tx_ctx.head_wb_addr = ring->dma +
2922                               (ring->count * sizeof(struct i40e_tx_desc));
2923
2924         /* As part of VSI creation/update, FW allocates certain
2925          * Tx arbitration queue sets for each TC enabled for
2926          * the VSI. The FW returns the handles to these queue
2927          * sets as part of the response buffer to Add VSI,
2928          * Update VSI, etc. AQ commands. It is expected that
2929          * these queue set handles be associated with the Tx
2930          * queues by the driver as part of the TX queue context
2931          * initialization. This has to be done regardless of
2932          * DCB as by default everything is mapped to TC0.
2933          */
2934         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2935         tx_ctx.rdylist_act = 0;
2936
2937         /* clear the context in the HMC */
2938         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2939         if (err) {
2940                 dev_info(&vsi->back->pdev->dev,
2941                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2942                          ring->queue_index, pf_q, err);
2943                 return -ENOMEM;
2944         }
2945
2946         /* set the context in the HMC */
2947         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2948         if (err) {
2949                 dev_info(&vsi->back->pdev->dev,
2950                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2951                          ring->queue_index, pf_q, err);
2952                 return -ENOMEM;
2953         }
2954
2955         /* Now associate this queue with this PCI function */
2956         if (vsi->type == I40E_VSI_VMDQ2) {
2957                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2958                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2959                            I40E_QTX_CTL_VFVM_INDX_MASK;
2960         } else {
2961                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2962         }
2963
2964         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2965                     I40E_QTX_CTL_PF_INDX_MASK);
2966         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2967         i40e_flush(hw);
2968
2969         /* cache tail off for easier writes later */
2970         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2971
2972         return 0;
2973 }
2974
2975 /**
2976  * i40e_configure_rx_ring - Configure a receive ring context
2977  * @ring: The Rx ring to configure
2978  *
2979  * Configure the Rx descriptor ring in the HMC context.
2980  **/
2981 static int i40e_configure_rx_ring(struct i40e_ring *ring)
2982 {
2983         struct i40e_vsi *vsi = ring->vsi;
2984         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
2985         u16 pf_q = vsi->base_queue + ring->queue_index;
2986         struct i40e_hw *hw = &vsi->back->hw;
2987         struct i40e_hmc_obj_rxq rx_ctx;
2988         i40e_status err = 0;
2989
2990         ring->state = 0;
2991
2992         /* clear the context structure first */
2993         memset(&rx_ctx, 0, sizeof(rx_ctx));
2994
2995         ring->rx_buf_len = vsi->rx_buf_len;
2996
2997         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
2998                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
2999
3000         rx_ctx.base = (ring->dma / 128);
3001         rx_ctx.qlen = ring->count;
3002
3003         /* use 32 byte descriptors */
3004         rx_ctx.dsize = 1;
3005
3006         /* descriptor type is always zero
3007          * rx_ctx.dtype = 0;
3008          */
3009         rx_ctx.hsplit_0 = 0;
3010
3011         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3012         if (hw->revision_id == 0)
3013                 rx_ctx.lrxqthresh = 0;
3014         else
3015                 rx_ctx.lrxqthresh = 2;
3016         rx_ctx.crcstrip = 1;
3017         rx_ctx.l2tsel = 1;
3018         /* this controls whether VLAN is stripped from inner headers */
3019         rx_ctx.showiv = 0;
3020         /* set the prefena field to 1 because the manual says to */
3021         rx_ctx.prefena = 1;
3022
3023         /* clear the context in the HMC */
3024         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3025         if (err) {
3026                 dev_info(&vsi->back->pdev->dev,
3027                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3028                          ring->queue_index, pf_q, err);
3029                 return -ENOMEM;
3030         }
3031
3032         /* set the context in the HMC */
3033         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3034         if (err) {
3035                 dev_info(&vsi->back->pdev->dev,
3036                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3037                          ring->queue_index, pf_q, err);
3038                 return -ENOMEM;
3039         }
3040
3041         /* configure Rx buffer alignment */
3042         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3043                 clear_ring_build_skb_enabled(ring);
3044         else
3045                 set_ring_build_skb_enabled(ring);
3046
3047         /* cache tail for quicker writes, and clear the reg before use */
3048         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3049         writel(0, ring->tail);
3050
3051         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3052
3053         return 0;
3054 }
3055
3056 /**
3057  * i40e_vsi_configure_tx - Configure the VSI for Tx
3058  * @vsi: VSI structure describing this set of rings and resources
3059  *
3060  * Configure the Tx VSI for operation.
3061  **/
3062 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3063 {
3064         int err = 0;
3065         u16 i;
3066
3067         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3068                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3069
3070         return err;
3071 }
3072
3073 /**
3074  * i40e_vsi_configure_rx - Configure the VSI for Rx
3075  * @vsi: the VSI being configured
3076  *
3077  * Configure the Rx VSI for operation.
3078  **/
3079 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3080 {
3081         int err = 0;
3082         u16 i;
3083
3084         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3085                 vsi->max_frame = I40E_MAX_RXBUFFER;
3086                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3087 #if (PAGE_SIZE < 8192)
3088         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3089                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3090                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3091                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3092 #endif
3093         } else {
3094                 vsi->max_frame = I40E_MAX_RXBUFFER;
3095                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3096                                                        I40E_RXBUFFER_2048;
3097         }
3098
3099         /* set up individual rings */
3100         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3101                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3102
3103         return err;
3104 }
3105
3106 /**
3107  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3108  * @vsi: ptr to the VSI
3109  **/
3110 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3111 {
3112         struct i40e_ring *tx_ring, *rx_ring;
3113         u16 qoffset, qcount;
3114         int i, n;
3115
3116         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3117                 /* Reset the TC information */
3118                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3119                         rx_ring = vsi->rx_rings[i];
3120                         tx_ring = vsi->tx_rings[i];
3121                         rx_ring->dcb_tc = 0;
3122                         tx_ring->dcb_tc = 0;
3123                 }
3124         }
3125
3126         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3127                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3128                         continue;
3129
3130                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3131                 qcount = vsi->tc_config.tc_info[n].qcount;
3132                 for (i = qoffset; i < (qoffset + qcount); i++) {
3133                         rx_ring = vsi->rx_rings[i];
3134                         tx_ring = vsi->tx_rings[i];
3135                         rx_ring->dcb_tc = n;
3136                         tx_ring->dcb_tc = n;
3137                 }
3138         }
3139 }
3140
3141 /**
3142  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3143  * @vsi: ptr to the VSI
3144  **/
3145 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3146 {
3147         struct i40e_pf *pf = vsi->back;
3148         int err;
3149
3150         if (vsi->netdev)
3151                 i40e_set_rx_mode(vsi->netdev);
3152
3153         if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
3154                 err = i40e_macaddr_init(vsi, pf->hw.mac.addr);
3155                 if (err) {
3156                         dev_warn(&pf->pdev->dev,
3157                                  "could not set up macaddr; err %d\n", err);
3158                 }
3159         }
3160 }
3161
3162 /**
3163  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3164  * @vsi: Pointer to the targeted VSI
3165  *
3166  * This function replays the hlist on the hw where all the SB Flow Director
3167  * filters were saved.
3168  **/
3169 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3170 {
3171         struct i40e_fdir_filter *filter;
3172         struct i40e_pf *pf = vsi->back;
3173         struct hlist_node *node;
3174
3175         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3176                 return;
3177
3178         /* Reset FDir counters as we're replaying all existing filters */
3179         pf->fd_tcp4_filter_cnt = 0;
3180         pf->fd_udp4_filter_cnt = 0;
3181         pf->fd_sctp4_filter_cnt = 0;
3182         pf->fd_ip4_filter_cnt = 0;
3183
3184         hlist_for_each_entry_safe(filter, node,
3185                                   &pf->fdir_filter_list, fdir_node) {
3186                 i40e_add_del_fdir(vsi, filter, true);
3187         }
3188 }
3189
3190 /**
3191  * i40e_vsi_configure - Set up the VSI for action
3192  * @vsi: the VSI being configured
3193  **/
3194 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3195 {
3196         int err;
3197
3198         i40e_set_vsi_rx_mode(vsi);
3199         i40e_restore_vlan(vsi);
3200         i40e_vsi_config_dcb_rings(vsi);
3201         err = i40e_vsi_configure_tx(vsi);
3202         if (!err)
3203                 err = i40e_vsi_configure_rx(vsi);
3204
3205         return err;
3206 }
3207
3208 /**
3209  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3210  * @vsi: the VSI being configured
3211  **/
3212 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3213 {
3214         struct i40e_pf *pf = vsi->back;
3215         struct i40e_hw *hw = &pf->hw;
3216         u16 vector;
3217         int i, q;
3218         u32 qp;
3219
3220         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3221          * and PFINT_LNKLSTn registers, e.g.:
3222          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3223          */
3224         qp = vsi->base_queue;
3225         vector = vsi->base_vector;
3226         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3227                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3228
3229                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3230                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3231                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3232                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3233                      q_vector->rx.itr);
3234                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3235                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3236                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3237                      q_vector->tx.itr);
3238                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3239                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3240
3241                 /* Linked list for the queuepairs assigned to this vector */
3242                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3243                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3244                         u32 val;
3245
3246                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3247                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3248                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3249                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3250                               (I40E_QUEUE_TYPE_TX
3251                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3252
3253                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3254
3255                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3256                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
3257                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3258                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
3259                               (I40E_QUEUE_TYPE_RX
3260                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3261
3262                         /* Terminate the linked list */
3263                         if (q == (q_vector->num_ringpairs - 1))
3264                                 val |= (I40E_QUEUE_END_OF_LIST
3265                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3266
3267                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3268                         qp++;
3269                 }
3270         }
3271
3272         i40e_flush(hw);
3273 }
3274
3275 /**
3276  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3277  * @hw: ptr to the hardware info
3278  **/
3279 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3280 {
3281         struct i40e_hw *hw = &pf->hw;
3282         u32 val;
3283
3284         /* clear things first */
3285         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3286         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3287
3288         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3289               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3290               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3291               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3292               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3293               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3294               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3295               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3296
3297         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3298                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3299
3300         if (pf->flags & I40E_FLAG_PTP)
3301                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3302
3303         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3304
3305         /* SW_ITR_IDX = 0, but don't change INTENA */
3306         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3307                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3308
3309         /* OTHER_ITR_IDX = 0 */
3310         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3311 }
3312
3313 /**
3314  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3315  * @vsi: the VSI being configured
3316  **/
3317 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3318 {
3319         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3320         struct i40e_pf *pf = vsi->back;
3321         struct i40e_hw *hw = &pf->hw;
3322         u32 val;
3323
3324         /* set the ITR configuration */
3325         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3326         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3327         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3328         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3329         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3330         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3331         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3332
3333         i40e_enable_misc_int_causes(pf);
3334
3335         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3336         wr32(hw, I40E_PFINT_LNKLST0, 0);
3337
3338         /* Associate the queue pair to the vector and enable the queue int */
3339         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
3340               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3341               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3342
3343         wr32(hw, I40E_QINT_RQCTL(0), val);
3344
3345         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3346               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3347               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3348
3349         wr32(hw, I40E_QINT_TQCTL(0), val);
3350         i40e_flush(hw);
3351 }
3352
3353 /**
3354  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3355  * @pf: board private structure
3356  **/
3357 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3358 {
3359         struct i40e_hw *hw = &pf->hw;
3360
3361         wr32(hw, I40E_PFINT_DYN_CTL0,
3362              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3363         i40e_flush(hw);
3364 }
3365
3366 /**
3367  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3368  * @pf: board private structure
3369  * @clearpba: true when all pending interrupt events should be cleared
3370  **/
3371 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3372 {
3373         struct i40e_hw *hw = &pf->hw;
3374         u32 val;
3375
3376         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3377               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3378               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3379
3380         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3381         i40e_flush(hw);
3382 }
3383
3384 /**
3385  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3386  * @irq: interrupt number
3387  * @data: pointer to a q_vector
3388  **/
3389 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3390 {
3391         struct i40e_q_vector *q_vector = data;
3392
3393         if (!q_vector->tx.ring && !q_vector->rx.ring)
3394                 return IRQ_HANDLED;
3395
3396         napi_schedule_irqoff(&q_vector->napi);
3397
3398         return IRQ_HANDLED;
3399 }
3400
3401 /**
3402  * i40e_irq_affinity_notify - Callback for affinity changes
3403  * @notify: context as to what irq was changed
3404  * @mask: the new affinity mask
3405  *
3406  * This is a callback function used by the irq_set_affinity_notifier function
3407  * so that we may register to receive changes to the irq affinity masks.
3408  **/
3409 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3410                                      const cpumask_t *mask)
3411 {
3412         struct i40e_q_vector *q_vector =
3413                 container_of(notify, struct i40e_q_vector, affinity_notify);
3414
3415         q_vector->affinity_mask = *mask;
3416 }
3417
3418 /**
3419  * i40e_irq_affinity_release - Callback for affinity notifier release
3420  * @ref: internal core kernel usage
3421  *
3422  * This is a callback function used by the irq_set_affinity_notifier function
3423  * to inform the current notification subscriber that they will no longer
3424  * receive notifications.
3425  **/
3426 static void i40e_irq_affinity_release(struct kref *ref) {}
3427
3428 /**
3429  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3430  * @vsi: the VSI being configured
3431  * @basename: name for the vector
3432  *
3433  * Allocates MSI-X vectors and requests interrupts from the kernel.
3434  **/
3435 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3436 {
3437         int q_vectors = vsi->num_q_vectors;
3438         struct i40e_pf *pf = vsi->back;
3439         int base = vsi->base_vector;
3440         int rx_int_idx = 0;
3441         int tx_int_idx = 0;
3442         int vector, err;
3443         int irq_num;
3444
3445         for (vector = 0; vector < q_vectors; vector++) {
3446                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3447
3448                 irq_num = pf->msix_entries[base + vector].vector;
3449
3450                 if (q_vector->tx.ring && q_vector->rx.ring) {
3451                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3452                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3453                         tx_int_idx++;
3454                 } else if (q_vector->rx.ring) {
3455                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3456                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3457                 } else if (q_vector->tx.ring) {
3458                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3459                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3460                 } else {
3461                         /* skip this unused q_vector */
3462                         continue;
3463                 }
3464                 err = request_irq(irq_num,
3465                                   vsi->irq_handler,
3466                                   0,
3467                                   q_vector->name,
3468                                   q_vector);
3469                 if (err) {
3470                         dev_info(&pf->pdev->dev,
3471                                  "MSIX request_irq failed, error: %d\n", err);
3472                         goto free_queue_irqs;
3473                 }
3474
3475                 /* register for affinity change notifications */
3476                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3477                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3478                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3479                 /* assign the mask for this irq */
3480                 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
3481         }
3482
3483         vsi->irqs_ready = true;
3484         return 0;
3485
3486 free_queue_irqs:
3487         while (vector) {
3488                 vector--;
3489                 irq_num = pf->msix_entries[base + vector].vector;
3490                 irq_set_affinity_notifier(irq_num, NULL);
3491                 irq_set_affinity_hint(irq_num, NULL);
3492                 free_irq(irq_num, &vsi->q_vectors[vector]);
3493         }
3494         return err;
3495 }
3496
3497 /**
3498  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3499  * @vsi: the VSI being un-configured
3500  **/
3501 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3502 {
3503         struct i40e_pf *pf = vsi->back;
3504         struct i40e_hw *hw = &pf->hw;
3505         int base = vsi->base_vector;
3506         int i;
3507
3508         for (i = 0; i < vsi->num_queue_pairs; i++) {
3509                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
3510                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
3511         }
3512
3513         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3514                 for (i = vsi->base_vector;
3515                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3516                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3517
3518                 i40e_flush(hw);
3519                 for (i = 0; i < vsi->num_q_vectors; i++)
3520                         synchronize_irq(pf->msix_entries[i + base].vector);
3521         } else {
3522                 /* Legacy and MSI mode - this stops all interrupt handling */
3523                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3524                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3525                 i40e_flush(hw);
3526                 synchronize_irq(pf->pdev->irq);
3527         }
3528 }
3529
3530 /**
3531  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3532  * @vsi: the VSI being configured
3533  **/
3534 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3535 {
3536         struct i40e_pf *pf = vsi->back;
3537         int i;
3538
3539         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3540                 for (i = 0; i < vsi->num_q_vectors; i++)
3541                         i40e_irq_dynamic_enable(vsi, i);
3542         } else {
3543                 i40e_irq_dynamic_enable_icr0(pf, true);
3544         }
3545
3546         i40e_flush(&pf->hw);
3547         return 0;
3548 }
3549
3550 /**
3551  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3552  * @pf: board private structure
3553  **/
3554 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3555 {
3556         /* Disable ICR 0 */
3557         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3558         i40e_flush(&pf->hw);
3559 }
3560
3561 /**
3562  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3563  * @irq: interrupt number
3564  * @data: pointer to a q_vector
3565  *
3566  * This is the handler used for all MSI/Legacy interrupts, and deals
3567  * with both queue and non-queue interrupts.  This is also used in
3568  * MSIX mode to handle the non-queue interrupts.
3569  **/
3570 static irqreturn_t i40e_intr(int irq, void *data)
3571 {
3572         struct i40e_pf *pf = (struct i40e_pf *)data;
3573         struct i40e_hw *hw = &pf->hw;
3574         irqreturn_t ret = IRQ_NONE;
3575         u32 icr0, icr0_remaining;
3576         u32 val, ena_mask;
3577
3578         icr0 = rd32(hw, I40E_PFINT_ICR0);
3579         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3580
3581         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3582         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3583                 goto enable_intr;
3584
3585         /* if interrupt but no bits showing, must be SWINT */
3586         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3587             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3588                 pf->sw_int_count++;
3589
3590         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3591             (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3592                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3593                 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3594                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3595         }
3596
3597         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3598         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3599                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3600                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3601
3602                 /* We do not have a way to disarm Queue causes while leaving
3603                  * interrupt enabled for all other causes, ideally
3604                  * interrupt should be disabled while we are in NAPI but
3605                  * this is not a performance path and napi_schedule()
3606                  * can deal with rescheduling.
3607                  */
3608                 if (!test_bit(__I40E_DOWN, &pf->state))
3609                         napi_schedule_irqoff(&q_vector->napi);
3610         }
3611
3612         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3613                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3614                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
3615                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3616         }
3617
3618         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3619                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3620                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
3621         }
3622
3623         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3624                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3625                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
3626         }
3627
3628         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3629                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
3630                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
3631                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3632                 val = rd32(hw, I40E_GLGEN_RSTAT);
3633                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3634                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3635                 if (val == I40E_RESET_CORER) {
3636                         pf->corer_count++;
3637                 } else if (val == I40E_RESET_GLOBR) {
3638                         pf->globr_count++;
3639                 } else if (val == I40E_RESET_EMPR) {
3640                         pf->empr_count++;
3641                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state);
3642                 }
3643         }
3644
3645         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3646                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3647                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3648                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3649                          rd32(hw, I40E_PFHMC_ERRORINFO),
3650                          rd32(hw, I40E_PFHMC_ERRORDATA));
3651         }
3652
3653         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3654                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3655
3656                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3657                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3658                         i40e_ptp_tx_hwtstamp(pf);
3659                 }
3660         }
3661
3662         /* If a critical error is pending we have no choice but to reset the
3663          * device.
3664          * Report and mask out any remaining unexpected interrupts.
3665          */
3666         icr0_remaining = icr0 & ena_mask;
3667         if (icr0_remaining) {
3668                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3669                          icr0_remaining);
3670                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3671                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3672                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3673                         dev_info(&pf->pdev->dev, "device will be reset\n");
3674                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
3675                         i40e_service_event_schedule(pf);
3676                 }
3677                 ena_mask &= ~icr0_remaining;
3678         }
3679         ret = IRQ_HANDLED;
3680
3681 enable_intr:
3682         /* re-enable interrupt causes */
3683         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3684         if (!test_bit(__I40E_DOWN, &pf->state)) {
3685                 i40e_service_event_schedule(pf);
3686                 i40e_irq_dynamic_enable_icr0(pf, false);
3687         }
3688
3689         return ret;
3690 }
3691
3692 /**
3693  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3694  * @tx_ring:  tx ring to clean
3695  * @budget:   how many cleans we're allowed
3696  *
3697  * Returns true if there's any budget left (e.g. the clean is finished)
3698  **/
3699 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3700 {
3701         struct i40e_vsi *vsi = tx_ring->vsi;
3702         u16 i = tx_ring->next_to_clean;
3703         struct i40e_tx_buffer *tx_buf;
3704         struct i40e_tx_desc *tx_desc;
3705
3706         tx_buf = &tx_ring->tx_bi[i];
3707         tx_desc = I40E_TX_DESC(tx_ring, i);
3708         i -= tx_ring->count;
3709
3710         do {
3711                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3712
3713                 /* if next_to_watch is not set then there is no work pending */
3714                 if (!eop_desc)
3715                         break;
3716
3717                 /* prevent any other reads prior to eop_desc */
3718                 read_barrier_depends();
3719
3720                 /* if the descriptor isn't done, no work yet to do */
3721                 if (!(eop_desc->cmd_type_offset_bsz &
3722                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3723                         break;
3724
3725                 /* clear next_to_watch to prevent false hangs */
3726                 tx_buf->next_to_watch = NULL;
3727
3728                 tx_desc->buffer_addr = 0;
3729                 tx_desc->cmd_type_offset_bsz = 0;
3730                 /* move past filter desc */
3731                 tx_buf++;
3732                 tx_desc++;
3733                 i++;
3734                 if (unlikely(!i)) {
3735                         i -= tx_ring->count;
3736                         tx_buf = tx_ring->tx_bi;
3737                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3738                 }
3739                 /* unmap skb header data */
3740                 dma_unmap_single(tx_ring->dev,
3741                                  dma_unmap_addr(tx_buf, dma),
3742                                  dma_unmap_len(tx_buf, len),
3743                                  DMA_TO_DEVICE);
3744                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3745                         kfree(tx_buf->raw_buf);
3746
3747                 tx_buf->raw_buf = NULL;
3748                 tx_buf->tx_flags = 0;
3749                 tx_buf->next_to_watch = NULL;
3750                 dma_unmap_len_set(tx_buf, len, 0);
3751                 tx_desc->buffer_addr = 0;
3752                 tx_desc->cmd_type_offset_bsz = 0;
3753
3754                 /* move us past the eop_desc for start of next FD desc */
3755                 tx_buf++;
3756                 tx_desc++;
3757                 i++;
3758                 if (unlikely(!i)) {
3759                         i -= tx_ring->count;
3760                         tx_buf = tx_ring->tx_bi;
3761                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3762                 }
3763
3764                 /* update budget accounting */
3765                 budget--;
3766         } while (likely(budget));
3767
3768         i += tx_ring->count;
3769         tx_ring->next_to_clean = i;
3770
3771         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3772                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3773
3774         return budget > 0;
3775 }
3776
3777 /**
3778  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3779  * @irq: interrupt number
3780  * @data: pointer to a q_vector
3781  **/
3782 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3783 {
3784         struct i40e_q_vector *q_vector = data;
3785         struct i40e_vsi *vsi;
3786
3787         if (!q_vector->tx.ring)
3788                 return IRQ_HANDLED;
3789
3790         vsi = q_vector->tx.ring->vsi;
3791         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3792
3793         return IRQ_HANDLED;
3794 }
3795
3796 /**
3797  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3798  * @vsi: the VSI being configured
3799  * @v_idx: vector index
3800  * @qp_idx: queue pair index
3801  **/
3802 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3803 {
3804         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3805         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3806         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3807
3808         tx_ring->q_vector = q_vector;
3809         tx_ring->next = q_vector->tx.ring;
3810         q_vector->tx.ring = tx_ring;
3811         q_vector->tx.count++;
3812
3813         rx_ring->q_vector = q_vector;
3814         rx_ring->next = q_vector->rx.ring;
3815         q_vector->rx.ring = rx_ring;
3816         q_vector->rx.count++;
3817 }
3818
3819 /**
3820  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3821  * @vsi: the VSI being configured
3822  *
3823  * This function maps descriptor rings to the queue-specific vectors
3824  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3825  * one vector per queue pair, but on a constrained vector budget, we
3826  * group the queue pairs as "efficiently" as possible.
3827  **/
3828 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3829 {
3830         int qp_remaining = vsi->num_queue_pairs;
3831         int q_vectors = vsi->num_q_vectors;
3832         int num_ringpairs;
3833         int v_start = 0;
3834         int qp_idx = 0;
3835
3836         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3837          * group them so there are multiple queues per vector.
3838          * It is also important to go through all the vectors available to be
3839          * sure that if we don't use all the vectors, that the remaining vectors
3840          * are cleared. This is especially important when decreasing the
3841          * number of queues in use.
3842          */
3843         for (; v_start < q_vectors; v_start++) {
3844                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3845
3846                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3847
3848                 q_vector->num_ringpairs = num_ringpairs;
3849
3850                 q_vector->rx.count = 0;
3851                 q_vector->tx.count = 0;
3852                 q_vector->rx.ring = NULL;
3853                 q_vector->tx.ring = NULL;
3854
3855                 while (num_ringpairs--) {
3856                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3857                         qp_idx++;
3858                         qp_remaining--;
3859                 }
3860         }
3861 }
3862
3863 /**
3864  * i40e_vsi_request_irq - Request IRQ from the OS
3865  * @vsi: the VSI being configured
3866  * @basename: name for the vector
3867  **/
3868 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3869 {
3870         struct i40e_pf *pf = vsi->back;
3871         int err;
3872
3873         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3874                 err = i40e_vsi_request_irq_msix(vsi, basename);
3875         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3876                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3877                                   pf->int_name, pf);
3878         else
3879                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3880                                   pf->int_name, pf);
3881
3882         if (err)
3883                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3884
3885         return err;
3886 }
3887
3888 #ifdef CONFIG_NET_POLL_CONTROLLER
3889 /**
3890  * i40e_netpoll - A Polling 'interrupt' handler
3891  * @netdev: network interface device structure
3892  *
3893  * This is used by netconsole to send skbs without having to re-enable
3894  * interrupts.  It's not called while the normal interrupt routine is executing.
3895  **/
3896 static void i40e_netpoll(struct net_device *netdev)
3897 {
3898         struct i40e_netdev_priv *np = netdev_priv(netdev);
3899         struct i40e_vsi *vsi = np->vsi;
3900         struct i40e_pf *pf = vsi->back;
3901         int i;
3902
3903         /* if interface is down do nothing */
3904         if (test_bit(__I40E_DOWN, &vsi->state))
3905                 return;
3906
3907         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3908                 for (i = 0; i < vsi->num_q_vectors; i++)
3909                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3910         } else {
3911                 i40e_intr(pf->pdev->irq, netdev);
3912         }
3913 }
3914 #endif
3915
3916 /**
3917  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3918  * @pf: the PF being configured
3919  * @pf_q: the PF queue
3920  * @enable: enable or disable state of the queue
3921  *
3922  * This routine will wait for the given Tx queue of the PF to reach the
3923  * enabled or disabled state.
3924  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3925  * multiple retries; else will return 0 in case of success.
3926  **/
3927 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3928 {
3929         int i;
3930         u32 tx_reg;
3931
3932         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3933                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
3934                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3935                         break;
3936
3937                 usleep_range(10, 20);
3938         }
3939         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3940                 return -ETIMEDOUT;
3941
3942         return 0;
3943 }
3944
3945 /**
3946  * i40e_vsi_control_tx - Start or stop a VSI's rings
3947  * @vsi: the VSI being configured
3948  * @enable: start or stop the rings
3949  **/
3950 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3951 {
3952         struct i40e_pf *pf = vsi->back;
3953         struct i40e_hw *hw = &pf->hw;
3954         int i, j, pf_q, ret = 0;
3955         u32 tx_reg;
3956
3957         pf_q = vsi->base_queue;
3958         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3959
3960                 /* warn the TX unit of coming changes */
3961                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
3962                 if (!enable)
3963                         usleep_range(10, 20);
3964
3965                 for (j = 0; j < 50; j++) {
3966                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
3967                         if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
3968                             ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
3969                                 break;
3970                         usleep_range(1000, 2000);
3971                 }
3972                 /* Skip if the queue is already in the requested state */
3973                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3974                         continue;
3975
3976                 /* turn on/off the queue */
3977                 if (enable) {
3978                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
3979                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
3980                 } else {
3981                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3982                 }
3983
3984                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
3985                 /* No waiting for the Tx queue to disable */
3986                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
3987                         continue;
3988
3989                 /* wait for the change to finish */
3990                 ret = i40e_pf_txq_wait(pf, pf_q, enable);
3991                 if (ret) {
3992                         dev_info(&pf->pdev->dev,
3993                                  "VSI seid %d Tx ring %d %sable timeout\n",
3994                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
3995                         break;
3996                 }
3997         }
3998
3999         return ret;
4000 }
4001
4002 /**
4003  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4004  * @pf: the PF being configured
4005  * @pf_q: the PF queue
4006  * @enable: enable or disable state of the queue
4007  *
4008  * This routine will wait for the given Rx queue of the PF to reach the
4009  * enabled or disabled state.
4010  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4011  * multiple retries; else will return 0 in case of success.
4012  **/
4013 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4014 {
4015         int i;
4016         u32 rx_reg;
4017
4018         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4019                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4020                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4021                         break;
4022
4023                 usleep_range(10, 20);
4024         }
4025         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4026                 return -ETIMEDOUT;
4027
4028         return 0;
4029 }
4030
4031 /**
4032  * i40e_vsi_control_rx - Start or stop a VSI's rings
4033  * @vsi: the VSI being configured
4034  * @enable: start or stop the rings
4035  **/
4036 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4037 {
4038         struct i40e_pf *pf = vsi->back;
4039         struct i40e_hw *hw = &pf->hw;
4040         int i, j, pf_q, ret = 0;
4041         u32 rx_reg;
4042
4043         pf_q = vsi->base_queue;
4044         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4045                 for (j = 0; j < 50; j++) {
4046                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4047                         if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4048                             ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4049                                 break;
4050                         usleep_range(1000, 2000);
4051                 }
4052
4053                 /* Skip if the queue is already in the requested state */
4054                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4055                         continue;
4056
4057                 /* turn on/off the queue */
4058                 if (enable)
4059                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4060                 else
4061                         rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4062                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4063                 /* No waiting for the Tx queue to disable */
4064                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
4065                         continue;
4066
4067                 /* wait for the change to finish */
4068                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4069                 if (ret) {
4070                         dev_info(&pf->pdev->dev,
4071                                  "VSI seid %d Rx ring %d %sable timeout\n",
4072                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4073                         break;
4074                 }
4075         }
4076
4077         /* Due to HW errata, on Rx disable only, the register can indicate done
4078          * before it really is. Needs 50ms to be sure
4079          */
4080         if (!enable)
4081                 mdelay(50);
4082
4083         return ret;
4084 }
4085
4086 /**
4087  * i40e_vsi_start_rings - Start a VSI's rings
4088  * @vsi: the VSI being configured
4089  **/
4090 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4091 {
4092         int ret = 0;
4093
4094         /* do rx first for enable and last for disable */
4095         ret = i40e_vsi_control_rx(vsi, true);
4096         if (ret)
4097                 return ret;
4098         ret = i40e_vsi_control_tx(vsi, true);
4099
4100         return ret;
4101 }
4102
4103 /**
4104  * i40e_vsi_stop_rings - Stop a VSI's rings
4105  * @vsi: the VSI being configured
4106  **/
4107 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4108 {
4109         /* do rx first for enable and last for disable
4110          * Ignore return value, we need to shutdown whatever we can
4111          */
4112         i40e_vsi_control_tx(vsi, false);
4113         i40e_vsi_control_rx(vsi, false);
4114 }
4115
4116 /**
4117  * i40e_vsi_free_irq - Free the irq association with the OS
4118  * @vsi: the VSI being configured
4119  **/
4120 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4121 {
4122         struct i40e_pf *pf = vsi->back;
4123         struct i40e_hw *hw = &pf->hw;
4124         int base = vsi->base_vector;
4125         u32 val, qp;
4126         int i;
4127
4128         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4129                 if (!vsi->q_vectors)
4130                         return;
4131
4132                 if (!vsi->irqs_ready)
4133                         return;
4134
4135                 vsi->irqs_ready = false;
4136                 for (i = 0; i < vsi->num_q_vectors; i++) {
4137                         int irq_num;
4138                         u16 vector;
4139
4140                         vector = i + base;
4141                         irq_num = pf->msix_entries[vector].vector;
4142
4143                         /* free only the irqs that were actually requested */
4144                         if (!vsi->q_vectors[i] ||
4145                             !vsi->q_vectors[i]->num_ringpairs)
4146                                 continue;
4147
4148                         /* clear the affinity notifier in the IRQ descriptor */
4149                         irq_set_affinity_notifier(irq_num, NULL);
4150                         /* clear the affinity_mask in the IRQ descriptor */
4151                         irq_set_affinity_hint(irq_num, NULL);
4152                         synchronize_irq(irq_num);
4153                         free_irq(irq_num, vsi->q_vectors[i]);
4154
4155                         /* Tear down the interrupt queue link list
4156                          *
4157                          * We know that they come in pairs and always
4158                          * the Rx first, then the Tx.  To clear the
4159                          * link list, stick the EOL value into the
4160                          * next_q field of the registers.
4161                          */
4162                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4163                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4164                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4165                         val |= I40E_QUEUE_END_OF_LIST
4166                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4167                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4168
4169                         while (qp != I40E_QUEUE_END_OF_LIST) {
4170                                 u32 next;
4171
4172                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4173
4174                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4175                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4176                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4177                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4178
4179                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4180                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4181
4182                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4183
4184                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4185
4186                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4187                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4188
4189                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4190                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4191                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4192                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4193
4194                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4195                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4196
4197                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4198                                 qp = next;
4199                         }
4200                 }
4201         } else {
4202                 free_irq(pf->pdev->irq, pf);
4203
4204                 val = rd32(hw, I40E_PFINT_LNKLST0);
4205                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4206                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4207                 val |= I40E_QUEUE_END_OF_LIST
4208                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4209                 wr32(hw, I40E_PFINT_LNKLST0, val);
4210
4211                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4212                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4213                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4214                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4215                          I40E_QINT_RQCTL_INTEVENT_MASK);
4216
4217                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4218                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4219
4220                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4221
4222                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4223
4224                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4225                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4226                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4227                          I40E_QINT_TQCTL_INTEVENT_MASK);
4228
4229                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4230                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4231
4232                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4233         }
4234 }
4235
4236 /**
4237  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4238  * @vsi: the VSI being configured
4239  * @v_idx: Index of vector to be freed
4240  *
4241  * This function frees the memory allocated to the q_vector.  In addition if
4242  * NAPI is enabled it will delete any references to the NAPI struct prior
4243  * to freeing the q_vector.
4244  **/
4245 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4246 {
4247         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4248         struct i40e_ring *ring;
4249
4250         if (!q_vector)
4251                 return;
4252
4253         /* disassociate q_vector from rings */
4254         i40e_for_each_ring(ring, q_vector->tx)
4255                 ring->q_vector = NULL;
4256
4257         i40e_for_each_ring(ring, q_vector->rx)
4258                 ring->q_vector = NULL;
4259
4260         /* only VSI w/ an associated netdev is set up w/ NAPI */
4261         if (vsi->netdev)
4262                 netif_napi_del(&q_vector->napi);
4263
4264         vsi->q_vectors[v_idx] = NULL;
4265
4266         kfree_rcu(q_vector, rcu);
4267 }
4268
4269 /**
4270  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4271  * @vsi: the VSI being un-configured
4272  *
4273  * This frees the memory allocated to the q_vectors and
4274  * deletes references to the NAPI struct.
4275  **/
4276 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4277 {
4278         int v_idx;
4279
4280         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4281                 i40e_free_q_vector(vsi, v_idx);
4282 }
4283
4284 /**
4285  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4286  * @pf: board private structure
4287  **/
4288 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4289 {
4290         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4291         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4292                 pci_disable_msix(pf->pdev);
4293                 kfree(pf->msix_entries);
4294                 pf->msix_entries = NULL;
4295                 kfree(pf->irq_pile);
4296                 pf->irq_pile = NULL;
4297         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4298                 pci_disable_msi(pf->pdev);
4299         }
4300         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4301 }
4302
4303 /**
4304  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4305  * @pf: board private structure
4306  *
4307  * We go through and clear interrupt specific resources and reset the structure
4308  * to pre-load conditions
4309  **/
4310 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4311 {
4312         int i;
4313
4314         i40e_stop_misc_vector(pf);
4315         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4316                 synchronize_irq(pf->msix_entries[0].vector);
4317                 free_irq(pf->msix_entries[0].vector, pf);
4318         }
4319
4320         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4321                       I40E_IWARP_IRQ_PILE_ID);
4322
4323         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4324         for (i = 0; i < pf->num_alloc_vsi; i++)
4325                 if (pf->vsi[i])
4326                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4327         i40e_reset_interrupt_capability(pf);
4328 }
4329
4330 /**
4331  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4332  * @vsi: the VSI being configured
4333  **/
4334 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4335 {
4336         int q_idx;
4337
4338         if (!vsi->netdev)
4339                 return;
4340
4341         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4342                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4343
4344                 if (q_vector->rx.ring || q_vector->tx.ring)
4345                         napi_enable(&q_vector->napi);
4346         }
4347 }
4348
4349 /**
4350  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4351  * @vsi: the VSI being configured
4352  **/
4353 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4354 {
4355         int q_idx;
4356
4357         if (!vsi->netdev)
4358                 return;
4359
4360         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4361                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4362
4363                 if (q_vector->rx.ring || q_vector->tx.ring)
4364                         napi_disable(&q_vector->napi);
4365         }
4366 }
4367
4368 /**
4369  * i40e_vsi_close - Shut down a VSI
4370  * @vsi: the vsi to be quelled
4371  **/
4372 static void i40e_vsi_close(struct i40e_vsi *vsi)
4373 {
4374         struct i40e_pf *pf = vsi->back;
4375         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
4376                 i40e_down(vsi);
4377         i40e_vsi_free_irq(vsi);
4378         i40e_vsi_free_tx_resources(vsi);
4379         i40e_vsi_free_rx_resources(vsi);
4380         vsi->current_netdev_flags = 0;
4381         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
4382         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
4383                 pf->flags |=  I40E_FLAG_CLIENT_RESET;
4384 }
4385
4386 /**
4387  * i40e_quiesce_vsi - Pause a given VSI
4388  * @vsi: the VSI being paused
4389  **/
4390 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4391 {
4392         if (test_bit(__I40E_DOWN, &vsi->state))
4393                 return;
4394
4395         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
4396         if (vsi->netdev && netif_running(vsi->netdev))
4397                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4398         else
4399                 i40e_vsi_close(vsi);
4400 }
4401
4402 /**
4403  * i40e_unquiesce_vsi - Resume a given VSI
4404  * @vsi: the VSI being resumed
4405  **/
4406 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4407 {
4408         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
4409                 return;
4410
4411         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4412         if (vsi->netdev && netif_running(vsi->netdev))
4413                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4414         else
4415                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4416 }
4417
4418 /**
4419  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4420  * @pf: the PF
4421  **/
4422 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4423 {
4424         int v;
4425
4426         for (v = 0; v < pf->num_alloc_vsi; v++) {
4427                 if (pf->vsi[v])
4428                         i40e_quiesce_vsi(pf->vsi[v]);
4429         }
4430 }
4431
4432 /**
4433  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4434  * @pf: the PF
4435  **/
4436 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4437 {
4438         int v;
4439
4440         for (v = 0; v < pf->num_alloc_vsi; v++) {
4441                 if (pf->vsi[v])
4442                         i40e_unquiesce_vsi(pf->vsi[v]);
4443         }
4444 }
4445
4446 #ifdef CONFIG_I40E_DCB
4447 /**
4448  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4449  * @vsi: the VSI being configured
4450  *
4451  * Wait until all queues on a given VSI have been disabled.
4452  **/
4453 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4454 {
4455         struct i40e_pf *pf = vsi->back;
4456         int i, pf_q, ret;
4457
4458         pf_q = vsi->base_queue;
4459         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4460                 /* Check and wait for the Tx queue */
4461                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4462                 if (ret) {
4463                         dev_info(&pf->pdev->dev,
4464                                  "VSI seid %d Tx ring %d disable timeout\n",
4465                                  vsi->seid, pf_q);
4466                         return ret;
4467                 }
4468                 /* Check and wait for the Tx queue */
4469                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4470                 if (ret) {
4471                         dev_info(&pf->pdev->dev,
4472                                  "VSI seid %d Rx ring %d disable timeout\n",
4473                                  vsi->seid, pf_q);
4474                         return ret;
4475                 }
4476         }
4477
4478         return 0;
4479 }
4480
4481 /**
4482  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4483  * @pf: the PF
4484  *
4485  * This function waits for the queues to be in disabled state for all the
4486  * VSIs that are managed by this PF.
4487  **/
4488 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4489 {
4490         int v, ret = 0;
4491
4492         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4493                 if (pf->vsi[v]) {
4494                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4495                         if (ret)
4496                                 break;
4497                 }
4498         }
4499
4500         return ret;
4501 }
4502
4503 #endif
4504
4505 /**
4506  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4507  * @q_idx: TX queue number
4508  * @vsi: Pointer to VSI struct
4509  *
4510  * This function checks specified queue for given VSI. Detects hung condition.
4511  * We proactively detect hung TX queues by checking if interrupts are disabled
4512  * but there are pending descriptors.  If it appears hung, attempt to recover
4513  * by triggering a SW interrupt.
4514  **/
4515 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4516 {
4517         struct i40e_ring *tx_ring = NULL;
4518         struct i40e_pf  *pf;
4519         u32 val, tx_pending;
4520         int i;
4521
4522         pf = vsi->back;
4523
4524         /* now that we have an index, find the tx_ring struct */
4525         for (i = 0; i < vsi->num_queue_pairs; i++) {
4526                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4527                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4528                                 tx_ring = vsi->tx_rings[i];
4529                                 break;
4530                         }
4531                 }
4532         }
4533
4534         if (!tx_ring)
4535                 return;
4536
4537         /* Read interrupt register */
4538         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4539                 val = rd32(&pf->hw,
4540                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4541                                                tx_ring->vsi->base_vector - 1));
4542         else
4543                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4544
4545         tx_pending = i40e_get_tx_pending(tx_ring);
4546
4547         /* Interrupts are disabled and TX pending is non-zero,
4548          * trigger the SW interrupt (don't wait). Worst case
4549          * there will be one extra interrupt which may result
4550          * into not cleaning any queues because queues are cleaned.
4551          */
4552         if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
4553                 i40e_force_wb(vsi, tx_ring->q_vector);
4554 }
4555
4556 /**
4557  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4558  * @pf:  pointer to PF struct
4559  *
4560  * LAN VSI has netdev and netdev has TX queues. This function is to check
4561  * each of those TX queues if they are hung, trigger recovery by issuing
4562  * SW interrupt.
4563  **/
4564 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4565 {
4566         struct net_device *netdev;
4567         struct i40e_vsi *vsi;
4568         int i;
4569
4570         /* Only for LAN VSI */
4571         vsi = pf->vsi[pf->lan_vsi];
4572
4573         if (!vsi)
4574                 return;
4575
4576         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4577         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
4578             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
4579                 return;
4580
4581         /* Make sure type is MAIN VSI */
4582         if (vsi->type != I40E_VSI_MAIN)
4583                 return;
4584
4585         netdev = vsi->netdev;
4586         if (!netdev)
4587                 return;
4588
4589         /* Bail out if netif_carrier is not OK */
4590         if (!netif_carrier_ok(netdev))
4591                 return;
4592
4593         /* Go thru' TX queues for netdev */
4594         for (i = 0; i < netdev->num_tx_queues; i++) {
4595                 struct netdev_queue *q;
4596
4597                 q = netdev_get_tx_queue(netdev, i);
4598                 if (q)
4599                         i40e_detect_recover_hung_queue(i, vsi);
4600         }
4601 }
4602
4603 /**
4604  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4605  * @pf: pointer to PF
4606  *
4607  * Get TC map for ISCSI PF type that will include iSCSI TC
4608  * and LAN TC.
4609  **/
4610 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4611 {
4612         struct i40e_dcb_app_priority_table app;
4613         struct i40e_hw *hw = &pf->hw;
4614         u8 enabled_tc = 1; /* TC0 is always enabled */
4615         u8 tc, i;
4616         /* Get the iSCSI APP TLV */
4617         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4618
4619         for (i = 0; i < dcbcfg->numapps; i++) {
4620                 app = dcbcfg->app[i];
4621                 if (app.selector == I40E_APP_SEL_TCPIP &&
4622                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4623                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4624                         enabled_tc |= BIT(tc);
4625                         break;
4626                 }
4627         }
4628
4629         return enabled_tc;
4630 }
4631
4632 /**
4633  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4634  * @dcbcfg: the corresponding DCBx configuration structure
4635  *
4636  * Return the number of TCs from given DCBx configuration
4637  **/
4638 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4639 {
4640         int i, tc_unused = 0;
4641         u8 num_tc = 0;
4642         u8 ret = 0;
4643
4644         /* Scan the ETS Config Priority Table to find
4645          * traffic class enabled for a given priority
4646          * and create a bitmask of enabled TCs
4647          */
4648         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4649                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4650
4651         /* Now scan the bitmask to check for
4652          * contiguous TCs starting with TC0
4653          */
4654         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4655                 if (num_tc & BIT(i)) {
4656                         if (!tc_unused) {
4657                                 ret++;
4658                         } else {
4659                                 pr_err("Non-contiguous TC - Disabling DCB\n");
4660                                 return 1;
4661                         }
4662                 } else {
4663                         tc_unused = 1;
4664                 }
4665         }
4666
4667         /* There is always at least TC0 */
4668         if (!ret)
4669                 ret = 1;
4670
4671         return ret;
4672 }
4673
4674 /**
4675  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4676  * @dcbcfg: the corresponding DCBx configuration structure
4677  *
4678  * Query the current DCB configuration and return the number of
4679  * traffic classes enabled from the given DCBX config
4680  **/
4681 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4682 {
4683         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4684         u8 enabled_tc = 1;
4685         u8 i;
4686
4687         for (i = 0; i < num_tc; i++)
4688                 enabled_tc |= BIT(i);
4689
4690         return enabled_tc;
4691 }
4692
4693 /**
4694  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4695  * @pf: PF being queried
4696  *
4697  * Return number of traffic classes enabled for the given PF
4698  **/
4699 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4700 {
4701         struct i40e_hw *hw = &pf->hw;
4702         u8 i, enabled_tc = 1;
4703         u8 num_tc = 0;
4704         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4705
4706         /* If DCB is not enabled then always in single TC */
4707         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4708                 return 1;
4709
4710         /* SFP mode will be enabled for all TCs on port */
4711         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4712                 return i40e_dcb_get_num_tc(dcbcfg);
4713
4714         /* MFP mode return count of enabled TCs for this PF */
4715         if (pf->hw.func_caps.iscsi)
4716                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4717         else
4718                 return 1; /* Only TC0 */
4719
4720         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4721                 if (enabled_tc & BIT(i))
4722                         num_tc++;
4723         }
4724         return num_tc;
4725 }
4726
4727 /**
4728  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4729  * @pf: PF being queried
4730  *
4731  * Return a bitmap for enabled traffic classes for this PF.
4732  **/
4733 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4734 {
4735         /* If DCB is not enabled for this PF then just return default TC */
4736         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4737                 return I40E_DEFAULT_TRAFFIC_CLASS;
4738
4739         /* SFP mode we want PF to be enabled for all TCs */
4740         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4741                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4742
4743         /* MFP enabled and iSCSI PF type */
4744         if (pf->hw.func_caps.iscsi)
4745                 return i40e_get_iscsi_tc_map(pf);
4746         else
4747                 return I40E_DEFAULT_TRAFFIC_CLASS;
4748 }
4749
4750 /**
4751  * i40e_vsi_get_bw_info - Query VSI BW Information
4752  * @vsi: the VSI being queried
4753  *
4754  * Returns 0 on success, negative value on failure
4755  **/
4756 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4757 {
4758         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4759         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4760         struct i40e_pf *pf = vsi->back;
4761         struct i40e_hw *hw = &pf->hw;
4762         i40e_status ret;
4763         u32 tc_bw_max;
4764         int i;
4765
4766         /* Get the VSI level BW configuration */
4767         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4768         if (ret) {
4769                 dev_info(&pf->pdev->dev,
4770                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4771                          i40e_stat_str(&pf->hw, ret),
4772                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4773                 return -EINVAL;
4774         }
4775
4776         /* Get the VSI level BW configuration per TC */
4777         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4778                                                NULL);
4779         if (ret) {
4780                 dev_info(&pf->pdev->dev,
4781                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4782                          i40e_stat_str(&pf->hw, ret),
4783                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4784                 return -EINVAL;
4785         }
4786
4787         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4788                 dev_info(&pf->pdev->dev,
4789                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4790                          bw_config.tc_valid_bits,
4791                          bw_ets_config.tc_valid_bits);
4792                 /* Still continuing */
4793         }
4794
4795         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4796         vsi->bw_max_quanta = bw_config.max_bw;
4797         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4798                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4799         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4800                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4801                 vsi->bw_ets_limit_credits[i] =
4802                                         le16_to_cpu(bw_ets_config.credits[i]);
4803                 /* 3 bits out of 4 for each TC */
4804                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4805         }
4806
4807         return 0;
4808 }
4809
4810 /**
4811  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4812  * @vsi: the VSI being configured
4813  * @enabled_tc: TC bitmap
4814  * @bw_credits: BW shared credits per TC
4815  *
4816  * Returns 0 on success, negative value on failure
4817  **/
4818 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4819                                        u8 *bw_share)
4820 {
4821         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4822         i40e_status ret;
4823         int i;
4824
4825         bw_data.tc_valid_bits = enabled_tc;
4826         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4827                 bw_data.tc_bw_credits[i] = bw_share[i];
4828
4829         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
4830                                        NULL);
4831         if (ret) {
4832                 dev_info(&vsi->back->pdev->dev,
4833                          "AQ command Config VSI BW allocation per TC failed = %d\n",
4834                          vsi->back->hw.aq.asq_last_status);
4835                 return -EINVAL;
4836         }
4837
4838         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4839                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
4840
4841         return 0;
4842 }
4843
4844 /**
4845  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
4846  * @vsi: the VSI being configured
4847  * @enabled_tc: TC map to be enabled
4848  *
4849  **/
4850 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4851 {
4852         struct net_device *netdev = vsi->netdev;
4853         struct i40e_pf *pf = vsi->back;
4854         struct i40e_hw *hw = &pf->hw;
4855         u8 netdev_tc = 0;
4856         int i;
4857         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4858
4859         if (!netdev)
4860                 return;
4861
4862         if (!enabled_tc) {
4863                 netdev_reset_tc(netdev);
4864                 return;
4865         }
4866
4867         /* Set up actual enabled TCs on the VSI */
4868         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
4869                 return;
4870
4871         /* set per TC queues for the VSI */
4872         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4873                 /* Only set TC queues for enabled tcs
4874                  *
4875                  * e.g. For a VSI that has TC0 and TC3 enabled the
4876                  * enabled_tc bitmap would be 0x00001001; the driver
4877                  * will set the numtc for netdev as 2 that will be
4878                  * referenced by the netdev layer as TC 0 and 1.
4879                  */
4880                 if (vsi->tc_config.enabled_tc & BIT(i))
4881                         netdev_set_tc_queue(netdev,
4882                                         vsi->tc_config.tc_info[i].netdev_tc,
4883                                         vsi->tc_config.tc_info[i].qcount,
4884                                         vsi->tc_config.tc_info[i].qoffset);
4885         }
4886
4887         /* Assign UP2TC map for the VSI */
4888         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
4889                 /* Get the actual TC# for the UP */
4890                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
4891                 /* Get the mapped netdev TC# for the UP */
4892                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
4893                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
4894         }
4895 }
4896
4897 /**
4898  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
4899  * @vsi: the VSI being configured
4900  * @ctxt: the ctxt buffer returned from AQ VSI update param command
4901  **/
4902 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
4903                                       struct i40e_vsi_context *ctxt)
4904 {
4905         /* copy just the sections touched not the entire info
4906          * since not all sections are valid as returned by
4907          * update vsi params
4908          */
4909         vsi->info.mapping_flags = ctxt->info.mapping_flags;
4910         memcpy(&vsi->info.queue_mapping,
4911                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
4912         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
4913                sizeof(vsi->info.tc_mapping));
4914 }
4915
4916 /**
4917  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
4918  * @vsi: VSI to be configured
4919  * @enabled_tc: TC bitmap
4920  *
4921  * This configures a particular VSI for TCs that are mapped to the
4922  * given TC bitmap. It uses default bandwidth share for TCs across
4923  * VSIs to configure TC for a particular VSI.
4924  *
4925  * NOTE:
4926  * It is expected that the VSI queues have been quisced before calling
4927  * this function.
4928  **/
4929 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4930 {
4931         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
4932         struct i40e_vsi_context ctxt;
4933         int ret = 0;
4934         int i;
4935
4936         /* Check if enabled_tc is same as existing or new TCs */
4937         if (vsi->tc_config.enabled_tc == enabled_tc)
4938                 return ret;
4939
4940         /* Enable ETS TCs with equal BW Share for now across all VSIs */
4941         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4942                 if (enabled_tc & BIT(i))
4943                         bw_share[i] = 1;
4944         }
4945
4946         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
4947         if (ret) {
4948                 dev_info(&vsi->back->pdev->dev,
4949                          "Failed configuring TC map %d for VSI %d\n",
4950                          enabled_tc, vsi->seid);
4951                 goto out;
4952         }
4953
4954         /* Update Queue Pairs Mapping for currently enabled UPs */
4955         ctxt.seid = vsi->seid;
4956         ctxt.pf_num = vsi->back->hw.pf_id;
4957         ctxt.vf_num = 0;
4958         ctxt.uplink_seid = vsi->uplink_seid;
4959         ctxt.info = vsi->info;
4960         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
4961
4962         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
4963                 ctxt.info.valid_sections |=
4964                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
4965                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
4966         }
4967
4968         /* Update the VSI after updating the VSI queue-mapping information */
4969         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
4970         if (ret) {
4971                 dev_info(&vsi->back->pdev->dev,
4972                          "Update vsi tc config failed, err %s aq_err %s\n",
4973                          i40e_stat_str(&vsi->back->hw, ret),
4974                          i40e_aq_str(&vsi->back->hw,
4975                                      vsi->back->hw.aq.asq_last_status));
4976                 goto out;
4977         }
4978         /* update the local VSI info with updated queue map */
4979         i40e_vsi_update_queue_map(vsi, &ctxt);
4980         vsi->info.valid_sections = 0;
4981
4982         /* Update current VSI BW information */
4983         ret = i40e_vsi_get_bw_info(vsi);
4984         if (ret) {
4985                 dev_info(&vsi->back->pdev->dev,
4986                          "Failed updating vsi bw info, err %s aq_err %s\n",
4987                          i40e_stat_str(&vsi->back->hw, ret),
4988                          i40e_aq_str(&vsi->back->hw,
4989                                      vsi->back->hw.aq.asq_last_status));
4990                 goto out;
4991         }
4992
4993         /* Update the netdev TC setup */
4994         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
4995 out:
4996         return ret;
4997 }
4998
4999 /**
5000  * i40e_veb_config_tc - Configure TCs for given VEB
5001  * @veb: given VEB
5002  * @enabled_tc: TC bitmap
5003  *
5004  * Configures given TC bitmap for VEB (switching) element
5005  **/
5006 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5007 {
5008         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5009         struct i40e_pf *pf = veb->pf;
5010         int ret = 0;
5011         int i;
5012
5013         /* No TCs or already enabled TCs just return */
5014         if (!enabled_tc || veb->enabled_tc == enabled_tc)
5015                 return ret;
5016
5017         bw_data.tc_valid_bits = enabled_tc;
5018         /* bw_data.absolute_credits is not set (relative) */
5019
5020         /* Enable ETS TCs with equal BW Share for now */
5021         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5022                 if (enabled_tc & BIT(i))
5023                         bw_data.tc_bw_share_credits[i] = 1;
5024         }
5025
5026         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5027                                                    &bw_data, NULL);
5028         if (ret) {
5029                 dev_info(&pf->pdev->dev,
5030                          "VEB bw config failed, err %s aq_err %s\n",
5031                          i40e_stat_str(&pf->hw, ret),
5032                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5033                 goto out;
5034         }
5035
5036         /* Update the BW information */
5037         ret = i40e_veb_get_bw_info(veb);
5038         if (ret) {
5039                 dev_info(&pf->pdev->dev,
5040                          "Failed getting veb bw config, err %s aq_err %s\n",
5041                          i40e_stat_str(&pf->hw, ret),
5042                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5043         }
5044
5045 out:
5046         return ret;
5047 }
5048
5049 #ifdef CONFIG_I40E_DCB
5050 /**
5051  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5052  * @pf: PF struct
5053  *
5054  * Reconfigure VEB/VSIs on a given PF; it is assumed that
5055  * the caller would've quiesce all the VSIs before calling
5056  * this function
5057  **/
5058 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5059 {
5060         u8 tc_map = 0;
5061         int ret;
5062         u8 v;
5063
5064         /* Enable the TCs available on PF to all VEBs */
5065         tc_map = i40e_pf_get_tc_map(pf);
5066         for (v = 0; v < I40E_MAX_VEB; v++) {
5067                 if (!pf->veb[v])
5068                         continue;
5069                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5070                 if (ret) {
5071                         dev_info(&pf->pdev->dev,
5072                                  "Failed configuring TC for VEB seid=%d\n",
5073                                  pf->veb[v]->seid);
5074                         /* Will try to configure as many components */
5075                 }
5076         }
5077
5078         /* Update each VSI */
5079         for (v = 0; v < pf->num_alloc_vsi; v++) {
5080                 if (!pf->vsi[v])
5081                         continue;
5082
5083                 /* - Enable all TCs for the LAN VSI
5084                  * - For all others keep them at TC0 for now
5085                  */
5086                 if (v == pf->lan_vsi)
5087                         tc_map = i40e_pf_get_tc_map(pf);
5088                 else
5089                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5090
5091                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5092                 if (ret) {
5093                         dev_info(&pf->pdev->dev,
5094                                  "Failed configuring TC for VSI seid=%d\n",
5095                                  pf->vsi[v]->seid);
5096                         /* Will try to configure as many components */
5097                 } else {
5098                         /* Re-configure VSI vectors based on updated TC map */
5099                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5100                         if (pf->vsi[v]->netdev)
5101                                 i40e_dcbnl_set_all(pf->vsi[v]);
5102                 }
5103         }
5104 }
5105
5106 /**
5107  * i40e_resume_port_tx - Resume port Tx
5108  * @pf: PF struct
5109  *
5110  * Resume a port's Tx and issue a PF reset in case of failure to
5111  * resume.
5112  **/
5113 static int i40e_resume_port_tx(struct i40e_pf *pf)
5114 {
5115         struct i40e_hw *hw = &pf->hw;
5116         int ret;
5117
5118         ret = i40e_aq_resume_port_tx(hw, NULL);
5119         if (ret) {
5120                 dev_info(&pf->pdev->dev,
5121                          "Resume Port Tx failed, err %s aq_err %s\n",
5122                           i40e_stat_str(&pf->hw, ret),
5123                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5124                 /* Schedule PF reset to recover */
5125                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5126                 i40e_service_event_schedule(pf);
5127         }
5128
5129         return ret;
5130 }
5131
5132 /**
5133  * i40e_init_pf_dcb - Initialize DCB configuration
5134  * @pf: PF being configured
5135  *
5136  * Query the current DCB configuration and cache it
5137  * in the hardware structure
5138  **/
5139 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5140 {
5141         struct i40e_hw *hw = &pf->hw;
5142         int err = 0;
5143
5144         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5145         if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT)
5146                 goto out;
5147
5148         /* Get the initial DCB configuration */
5149         err = i40e_init_dcb(hw);
5150         if (!err) {
5151                 /* Device/Function is not DCBX capable */
5152                 if ((!hw->func_caps.dcb) ||
5153                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5154                         dev_info(&pf->pdev->dev,
5155                                  "DCBX offload is not supported or is disabled for this PF.\n");
5156                 } else {
5157                         /* When status is not DISABLED then DCBX in FW */
5158                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5159                                        DCB_CAP_DCBX_VER_IEEE;
5160
5161                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5162                         /* Enable DCB tagging only when more than one TC
5163                          * or explicitly disable if only one TC
5164                          */
5165                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5166                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5167                         else
5168                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5169                         dev_dbg(&pf->pdev->dev,
5170                                 "DCBX offload is supported for this PF.\n");
5171                 }
5172         } else {
5173                 dev_info(&pf->pdev->dev,
5174                          "Query for DCB configuration failed, err %s aq_err %s\n",
5175                          i40e_stat_str(&pf->hw, err),
5176                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5177         }
5178
5179 out:
5180         return err;
5181 }
5182 #endif /* CONFIG_I40E_DCB */
5183 #define SPEED_SIZE 14
5184 #define FC_SIZE 8
5185 /**
5186  * i40e_print_link_message - print link up or down
5187  * @vsi: the VSI for which link needs a message
5188  */
5189 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5190 {
5191         enum i40e_aq_link_speed new_speed;
5192         char *speed = "Unknown";
5193         char *fc = "Unknown";
5194         char *fec = "";
5195         char *an = "";
5196
5197         new_speed = vsi->back->hw.phy.link_info.link_speed;
5198
5199         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5200                 return;
5201         vsi->current_isup = isup;
5202         vsi->current_speed = new_speed;
5203         if (!isup) {
5204                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5205                 return;
5206         }
5207
5208         /* Warn user if link speed on NPAR enabled partition is not at
5209          * least 10GB
5210          */
5211         if (vsi->back->hw.func_caps.npar_enable &&
5212             (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5213              vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5214                 netdev_warn(vsi->netdev,
5215                             "The partition detected link speed that is less than 10Gbps\n");
5216
5217         switch (vsi->back->hw.phy.link_info.link_speed) {
5218         case I40E_LINK_SPEED_40GB:
5219                 speed = "40 G";
5220                 break;
5221         case I40E_LINK_SPEED_20GB:
5222                 speed = "20 G";
5223                 break;
5224         case I40E_LINK_SPEED_25GB:
5225                 speed = "25 G";
5226                 break;
5227         case I40E_LINK_SPEED_10GB:
5228                 speed = "10 G";
5229                 break;
5230         case I40E_LINK_SPEED_1GB:
5231                 speed = "1000 M";
5232                 break;
5233         case I40E_LINK_SPEED_100MB:
5234                 speed = "100 M";
5235                 break;
5236         default:
5237                 break;
5238         }
5239
5240         switch (vsi->back->hw.fc.current_mode) {
5241         case I40E_FC_FULL:
5242                 fc = "RX/TX";
5243                 break;
5244         case I40E_FC_TX_PAUSE:
5245                 fc = "TX";
5246                 break;
5247         case I40E_FC_RX_PAUSE:
5248                 fc = "RX";
5249                 break;
5250         default:
5251                 fc = "None";
5252                 break;
5253         }
5254
5255         if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
5256                 fec = ", FEC: None";
5257                 an = ", Autoneg: False";
5258
5259                 if (vsi->back->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
5260                         an = ", Autoneg: True";
5261
5262                 if (vsi->back->hw.phy.link_info.fec_info &
5263                     I40E_AQ_CONFIG_FEC_KR_ENA)
5264                         fec = ", FEC: CL74 FC-FEC/BASE-R";
5265                 else if (vsi->back->hw.phy.link_info.fec_info &
5266                          I40E_AQ_CONFIG_FEC_RS_ENA)
5267                         fec = ", FEC: CL108 RS-FEC";
5268         }
5269
5270         netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s, Flow Control: %s\n",
5271                     speed, fec, an, fc);
5272 }
5273
5274 /**
5275  * i40e_up_complete - Finish the last steps of bringing up a connection
5276  * @vsi: the VSI being configured
5277  **/
5278 static int i40e_up_complete(struct i40e_vsi *vsi)
5279 {
5280         struct i40e_pf *pf = vsi->back;
5281         int err;
5282
5283         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5284                 i40e_vsi_configure_msix(vsi);
5285         else
5286                 i40e_configure_msi_and_legacy(vsi);
5287
5288         /* start rings */
5289         err = i40e_vsi_start_rings(vsi);
5290         if (err)
5291                 return err;
5292
5293         clear_bit(__I40E_DOWN, &vsi->state);
5294         i40e_napi_enable_all(vsi);
5295         i40e_vsi_enable_irq(vsi);
5296
5297         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5298             (vsi->netdev)) {
5299                 i40e_print_link_message(vsi, true);
5300                 netif_tx_start_all_queues(vsi->netdev);
5301                 netif_carrier_on(vsi->netdev);
5302         } else if (vsi->netdev) {
5303                 i40e_print_link_message(vsi, false);
5304                 /* need to check for qualified module here*/
5305                 if ((pf->hw.phy.link_info.link_info &
5306                         I40E_AQ_MEDIA_AVAILABLE) &&
5307                     (!(pf->hw.phy.link_info.an_info &
5308                         I40E_AQ_QUALIFIED_MODULE)))
5309                         netdev_err(vsi->netdev,
5310                                    "the driver failed to link because an unqualified module was detected.");
5311         }
5312
5313         /* replay FDIR SB filters */
5314         if (vsi->type == I40E_VSI_FDIR) {
5315                 /* reset fd counters */
5316                 pf->fd_add_err = 0;
5317                 pf->fd_atr_cnt = 0;
5318                 i40e_fdir_filter_restore(vsi);
5319         }
5320
5321         /* On the next run of the service_task, notify any clients of the new
5322          * opened netdev
5323          */
5324         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5325         i40e_service_event_schedule(pf);
5326
5327         return 0;
5328 }
5329
5330 /**
5331  * i40e_vsi_reinit_locked - Reset the VSI
5332  * @vsi: the VSI being configured
5333  *
5334  * Rebuild the ring structs after some configuration
5335  * has changed, e.g. MTU size.
5336  **/
5337 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5338 {
5339         struct i40e_pf *pf = vsi->back;
5340
5341         WARN_ON(in_interrupt());
5342         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
5343                 usleep_range(1000, 2000);
5344         i40e_down(vsi);
5345
5346         i40e_up(vsi);
5347         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
5348 }
5349
5350 /**
5351  * i40e_up - Bring the connection back up after being down
5352  * @vsi: the VSI being configured
5353  **/
5354 int i40e_up(struct i40e_vsi *vsi)
5355 {
5356         int err;
5357
5358         err = i40e_vsi_configure(vsi);
5359         if (!err)
5360                 err = i40e_up_complete(vsi);
5361
5362         return err;
5363 }
5364
5365 /**
5366  * i40e_down - Shutdown the connection processing
5367  * @vsi: the VSI being stopped
5368  **/
5369 void i40e_down(struct i40e_vsi *vsi)
5370 {
5371         int i;
5372
5373         /* It is assumed that the caller of this function
5374          * sets the vsi->state __I40E_DOWN bit.
5375          */
5376         if (vsi->netdev) {
5377                 netif_carrier_off(vsi->netdev);
5378                 netif_tx_disable(vsi->netdev);
5379         }
5380         i40e_vsi_disable_irq(vsi);
5381         i40e_vsi_stop_rings(vsi);
5382         i40e_napi_disable_all(vsi);
5383
5384         for (i = 0; i < vsi->num_queue_pairs; i++) {
5385                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5386                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5387         }
5388
5389 }
5390
5391 /**
5392  * i40e_setup_tc - configure multiple traffic classes
5393  * @netdev: net device to configure
5394  * @tc: number of traffic classes to enable
5395  **/
5396 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5397 {
5398         struct i40e_netdev_priv *np = netdev_priv(netdev);
5399         struct i40e_vsi *vsi = np->vsi;
5400         struct i40e_pf *pf = vsi->back;
5401         u8 enabled_tc = 0;
5402         int ret = -EINVAL;
5403         int i;
5404
5405         /* Check if DCB enabled to continue */
5406         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5407                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5408                 goto exit;
5409         }
5410
5411         /* Check if MFP enabled */
5412         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5413                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5414                 goto exit;
5415         }
5416
5417         /* Check whether tc count is within enabled limit */
5418         if (tc > i40e_pf_get_num_tc(pf)) {
5419                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5420                 goto exit;
5421         }
5422
5423         /* Generate TC map for number of tc requested */
5424         for (i = 0; i < tc; i++)
5425                 enabled_tc |= BIT(i);
5426
5427         /* Requesting same TC configuration as already enabled */
5428         if (enabled_tc == vsi->tc_config.enabled_tc)
5429                 return 0;
5430
5431         /* Quiesce VSI queues */
5432         i40e_quiesce_vsi(vsi);
5433
5434         /* Configure VSI for enabled TCs */
5435         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5436         if (ret) {
5437                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5438                             vsi->seid);
5439                 goto exit;
5440         }
5441
5442         /* Unquiesce VSI */
5443         i40e_unquiesce_vsi(vsi);
5444
5445 exit:
5446         return ret;
5447 }
5448
5449 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
5450                            struct tc_to_netdev *tc)
5451 {
5452         if (tc->type != TC_SETUP_MQPRIO)
5453                 return -EINVAL;
5454
5455         tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
5456
5457         return i40e_setup_tc(netdev, tc->mqprio->num_tc);
5458 }
5459
5460 /**
5461  * i40e_open - Called when a network interface is made active
5462  * @netdev: network interface device structure
5463  *
5464  * The open entry point is called when a network interface is made
5465  * active by the system (IFF_UP).  At this point all resources needed
5466  * for transmit and receive operations are allocated, the interrupt
5467  * handler is registered with the OS, the netdev watchdog subtask is
5468  * enabled, and the stack is notified that the interface is ready.
5469  *
5470  * Returns 0 on success, negative value on failure
5471  **/
5472 int i40e_open(struct net_device *netdev)
5473 {
5474         struct i40e_netdev_priv *np = netdev_priv(netdev);
5475         struct i40e_vsi *vsi = np->vsi;
5476         struct i40e_pf *pf = vsi->back;
5477         int err;
5478
5479         /* disallow open during test or if eeprom is broken */
5480         if (test_bit(__I40E_TESTING, &pf->state) ||
5481             test_bit(__I40E_BAD_EEPROM, &pf->state))
5482                 return -EBUSY;
5483
5484         netif_carrier_off(netdev);
5485
5486         err = i40e_vsi_open(vsi);
5487         if (err)
5488                 return err;
5489
5490         /* configure global TSO hardware offload settings */
5491         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5492                                                        TCP_FLAG_FIN) >> 16);
5493         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5494                                                        TCP_FLAG_FIN |
5495                                                        TCP_FLAG_CWR) >> 16);
5496         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5497
5498         udp_tunnel_get_rx_info(netdev);
5499
5500         return 0;
5501 }
5502
5503 /**
5504  * i40e_vsi_open -
5505  * @vsi: the VSI to open
5506  *
5507  * Finish initialization of the VSI.
5508  *
5509  * Returns 0 on success, negative value on failure
5510  *
5511  * Note: expects to be called while under rtnl_lock()
5512  **/
5513 int i40e_vsi_open(struct i40e_vsi *vsi)
5514 {
5515         struct i40e_pf *pf = vsi->back;
5516         char int_name[I40E_INT_NAME_STR_LEN];
5517         int err;
5518
5519         /* allocate descriptors */
5520         err = i40e_vsi_setup_tx_resources(vsi);
5521         if (err)
5522                 goto err_setup_tx;
5523         err = i40e_vsi_setup_rx_resources(vsi);
5524         if (err)
5525                 goto err_setup_rx;
5526
5527         err = i40e_vsi_configure(vsi);
5528         if (err)
5529                 goto err_setup_rx;
5530
5531         if (vsi->netdev) {
5532                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5533                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5534                 err = i40e_vsi_request_irq(vsi, int_name);
5535                 if (err)
5536                         goto err_setup_rx;
5537
5538                 /* Notify the stack of the actual queue counts. */
5539                 err = netif_set_real_num_tx_queues(vsi->netdev,
5540                                                    vsi->num_queue_pairs);
5541                 if (err)
5542                         goto err_set_queues;
5543
5544                 err = netif_set_real_num_rx_queues(vsi->netdev,
5545                                                    vsi->num_queue_pairs);
5546                 if (err)
5547                         goto err_set_queues;
5548
5549         } else if (vsi->type == I40E_VSI_FDIR) {
5550                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5551                          dev_driver_string(&pf->pdev->dev),
5552                          dev_name(&pf->pdev->dev));
5553                 err = i40e_vsi_request_irq(vsi, int_name);
5554
5555         } else {
5556                 err = -EINVAL;
5557                 goto err_setup_rx;
5558         }
5559
5560         err = i40e_up_complete(vsi);
5561         if (err)
5562                 goto err_up_complete;
5563
5564         return 0;
5565
5566 err_up_complete:
5567         i40e_down(vsi);
5568 err_set_queues:
5569         i40e_vsi_free_irq(vsi);
5570 err_setup_rx:
5571         i40e_vsi_free_rx_resources(vsi);
5572 err_setup_tx:
5573         i40e_vsi_free_tx_resources(vsi);
5574         if (vsi == pf->vsi[pf->lan_vsi])
5575                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
5576
5577         return err;
5578 }
5579
5580 /**
5581  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5582  * @pf: Pointer to PF
5583  *
5584  * This function destroys the hlist where all the Flow Director
5585  * filters were saved.
5586  **/
5587 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5588 {
5589         struct i40e_fdir_filter *filter;
5590         struct i40e_flex_pit *pit_entry, *tmp;
5591         struct hlist_node *node2;
5592
5593         hlist_for_each_entry_safe(filter, node2,
5594                                   &pf->fdir_filter_list, fdir_node) {
5595                 hlist_del(&filter->fdir_node);
5596                 kfree(filter);
5597         }
5598
5599         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
5600                 list_del(&pit_entry->list);
5601                 kfree(pit_entry);
5602         }
5603         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
5604
5605         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
5606                 list_del(&pit_entry->list);
5607                 kfree(pit_entry);
5608         }
5609         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
5610
5611         pf->fdir_pf_active_filters = 0;
5612         pf->fd_tcp4_filter_cnt = 0;
5613         pf->fd_udp4_filter_cnt = 0;
5614         pf->fd_sctp4_filter_cnt = 0;
5615         pf->fd_ip4_filter_cnt = 0;
5616
5617         /* Reprogram the default input set for TCP/IPv4 */
5618         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
5619                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5620                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5621
5622         /* Reprogram the default input set for UDP/IPv4 */
5623         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
5624                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5625                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5626
5627         /* Reprogram the default input set for SCTP/IPv4 */
5628         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
5629                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
5630                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
5631
5632         /* Reprogram the default input set for Other/IPv4 */
5633         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
5634                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
5635 }
5636
5637 /**
5638  * i40e_close - Disables a network interface
5639  * @netdev: network interface device structure
5640  *
5641  * The close entry point is called when an interface is de-activated
5642  * by the OS.  The hardware is still under the driver's control, but
5643  * this netdev interface is disabled.
5644  *
5645  * Returns 0, this is not allowed to fail
5646  **/
5647 int i40e_close(struct net_device *netdev)
5648 {
5649         struct i40e_netdev_priv *np = netdev_priv(netdev);
5650         struct i40e_vsi *vsi = np->vsi;
5651
5652         i40e_vsi_close(vsi);
5653
5654         return 0;
5655 }
5656
5657 /**
5658  * i40e_do_reset - Start a PF or Core Reset sequence
5659  * @pf: board private structure
5660  * @reset_flags: which reset is requested
5661  * @lock_acquired: indicates whether or not the lock has been acquired
5662  * before this function was called.
5663  *
5664  * The essential difference in resets is that the PF Reset
5665  * doesn't clear the packet buffers, doesn't reset the PE
5666  * firmware, and doesn't bother the other PFs on the chip.
5667  **/
5668 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
5669 {
5670         u32 val;
5671
5672         WARN_ON(in_interrupt());
5673
5674
5675         /* do the biggest reset indicated */
5676         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5677
5678                 /* Request a Global Reset
5679                  *
5680                  * This will start the chip's countdown to the actual full
5681                  * chip reset event, and a warning interrupt to be sent
5682                  * to all PFs, including the requestor.  Our handler
5683                  * for the warning interrupt will deal with the shutdown
5684                  * and recovery of the switch setup.
5685                  */
5686                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5687                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5688                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5689                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5690
5691         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5692
5693                 /* Request a Core Reset
5694                  *
5695                  * Same as Global Reset, except does *not* include the MAC/PHY
5696                  */
5697                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5698                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5699                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5700                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5701                 i40e_flush(&pf->hw);
5702
5703         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5704
5705                 /* Request a PF Reset
5706                  *
5707                  * Resets only the PF-specific registers
5708                  *
5709                  * This goes directly to the tear-down and rebuild of
5710                  * the switch, since we need to do all the recovery as
5711                  * for the Core Reset.
5712                  */
5713                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5714                 i40e_handle_reset_warning(pf, lock_acquired);
5715
5716         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5717                 int v;
5718
5719                 /* Find the VSI(s) that requested a re-init */
5720                 dev_info(&pf->pdev->dev,
5721                          "VSI reinit requested\n");
5722                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5723                         struct i40e_vsi *vsi = pf->vsi[v];
5724
5725                         if (vsi != NULL &&
5726                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
5727                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5728                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
5729                         }
5730                 }
5731         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5732                 int v;
5733
5734                 /* Find the VSI(s) that needs to be brought down */
5735                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5736                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5737                         struct i40e_vsi *vsi = pf->vsi[v];
5738
5739                         if (vsi != NULL &&
5740                             test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) {
5741                                 set_bit(__I40E_DOWN, &vsi->state);
5742                                 i40e_down(vsi);
5743                                 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state);
5744                         }
5745                 }
5746         } else {
5747                 dev_info(&pf->pdev->dev,
5748                          "bad reset request 0x%08x\n", reset_flags);
5749         }
5750 }
5751
5752 #ifdef CONFIG_I40E_DCB
5753 /**
5754  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5755  * @pf: board private structure
5756  * @old_cfg: current DCB config
5757  * @new_cfg: new DCB config
5758  **/
5759 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5760                             struct i40e_dcbx_config *old_cfg,
5761                             struct i40e_dcbx_config *new_cfg)
5762 {
5763         bool need_reconfig = false;
5764
5765         /* Check if ETS configuration has changed */
5766         if (memcmp(&new_cfg->etscfg,
5767                    &old_cfg->etscfg,
5768                    sizeof(new_cfg->etscfg))) {
5769                 /* If Priority Table has changed reconfig is needed */
5770                 if (memcmp(&new_cfg->etscfg.prioritytable,
5771                            &old_cfg->etscfg.prioritytable,
5772                            sizeof(new_cfg->etscfg.prioritytable))) {
5773                         need_reconfig = true;
5774                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5775                 }
5776
5777                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5778                            &old_cfg->etscfg.tcbwtable,
5779                            sizeof(new_cfg->etscfg.tcbwtable)))
5780                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5781
5782                 if (memcmp(&new_cfg->etscfg.tsatable,
5783                            &old_cfg->etscfg.tsatable,
5784                            sizeof(new_cfg->etscfg.tsatable)))
5785                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5786         }
5787
5788         /* Check if PFC configuration has changed */
5789         if (memcmp(&new_cfg->pfc,
5790                    &old_cfg->pfc,
5791                    sizeof(new_cfg->pfc))) {
5792                 need_reconfig = true;
5793                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5794         }
5795
5796         /* Check if APP Table has changed */
5797         if (memcmp(&new_cfg->app,
5798                    &old_cfg->app,
5799                    sizeof(new_cfg->app))) {
5800                 need_reconfig = true;
5801                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5802         }
5803
5804         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5805         return need_reconfig;
5806 }
5807
5808 /**
5809  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5810  * @pf: board private structure
5811  * @e: event info posted on ARQ
5812  **/
5813 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5814                                   struct i40e_arq_event_info *e)
5815 {
5816         struct i40e_aqc_lldp_get_mib *mib =
5817                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5818         struct i40e_hw *hw = &pf->hw;
5819         struct i40e_dcbx_config tmp_dcbx_cfg;
5820         bool need_reconfig = false;
5821         int ret = 0;
5822         u8 type;
5823
5824         /* Not DCB capable or capability disabled */
5825         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
5826                 return ret;
5827
5828         /* Ignore if event is not for Nearest Bridge */
5829         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
5830                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
5831         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
5832         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
5833                 return ret;
5834
5835         /* Check MIB Type and return if event for Remote MIB update */
5836         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
5837         dev_dbg(&pf->pdev->dev,
5838                 "LLDP event mib type %s\n", type ? "remote" : "local");
5839         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
5840                 /* Update the remote cached instance and return */
5841                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
5842                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
5843                                 &hw->remote_dcbx_config);
5844                 goto exit;
5845         }
5846
5847         /* Store the old configuration */
5848         tmp_dcbx_cfg = hw->local_dcbx_config;
5849
5850         /* Reset the old DCBx configuration data */
5851         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
5852         /* Get updated DCBX data from firmware */
5853         ret = i40e_get_dcb_config(&pf->hw);
5854         if (ret) {
5855                 dev_info(&pf->pdev->dev,
5856                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
5857                          i40e_stat_str(&pf->hw, ret),
5858                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5859                 goto exit;
5860         }
5861
5862         /* No change detected in DCBX configs */
5863         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
5864                     sizeof(tmp_dcbx_cfg))) {
5865                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
5866                 goto exit;
5867         }
5868
5869         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
5870                                                &hw->local_dcbx_config);
5871
5872         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
5873
5874         if (!need_reconfig)
5875                 goto exit;
5876
5877         /* Enable DCB tagging only when more than one TC */
5878         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5879                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5880         else
5881                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5882
5883         set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5884         /* Reconfiguration needed quiesce all VSIs */
5885         i40e_pf_quiesce_all_vsi(pf);
5886
5887         /* Changes in configuration update VEB/VSI */
5888         i40e_dcb_reconfigure(pf);
5889
5890         ret = i40e_resume_port_tx(pf);
5891
5892         clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5893         /* In case of error no point in resuming VSIs */
5894         if (ret)
5895                 goto exit;
5896
5897         /* Wait for the PF's queues to be disabled */
5898         ret = i40e_pf_wait_queues_disabled(pf);
5899         if (ret) {
5900                 /* Schedule PF reset to recover */
5901                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5902                 i40e_service_event_schedule(pf);
5903         } else {
5904                 i40e_pf_unquiesce_all_vsi(pf);
5905         pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED |
5906                       I40E_FLAG_CLIENT_L2_CHANGE);
5907         }
5908
5909 exit:
5910         return ret;
5911 }
5912 #endif /* CONFIG_I40E_DCB */
5913
5914 /**
5915  * i40e_do_reset_safe - Protected reset path for userland calls.
5916  * @pf: board private structure
5917  * @reset_flags: which reset is requested
5918  *
5919  **/
5920 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
5921 {
5922         rtnl_lock();
5923         i40e_do_reset(pf, reset_flags, true);
5924         rtnl_unlock();
5925 }
5926
5927 /**
5928  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
5929  * @pf: board private structure
5930  * @e: event info posted on ARQ
5931  *
5932  * Handler for LAN Queue Overflow Event generated by the firmware for PF
5933  * and VF queues
5934  **/
5935 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
5936                                            struct i40e_arq_event_info *e)
5937 {
5938         struct i40e_aqc_lan_overflow *data =
5939                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
5940         u32 queue = le32_to_cpu(data->prtdcb_rupto);
5941         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
5942         struct i40e_hw *hw = &pf->hw;
5943         struct i40e_vf *vf;
5944         u16 vf_id;
5945
5946         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
5947                 queue, qtx_ctl);
5948
5949         /* Queue belongs to VF, find the VF and issue VF reset */
5950         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
5951             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
5952                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
5953                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
5954                 vf_id -= hw->func_caps.vf_base_id;
5955                 vf = &pf->vf[vf_id];
5956                 i40e_vc_notify_vf_reset(vf);
5957                 /* Allow VF to process pending reset notification */
5958                 msleep(20);
5959                 i40e_reset_vf(vf, false);
5960         }
5961 }
5962
5963 /**
5964  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
5965  * @pf: board private structure
5966  **/
5967 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
5968 {
5969         u32 val, fcnt_prog;
5970
5971         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5972         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
5973         return fcnt_prog;
5974 }
5975
5976 /**
5977  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
5978  * @pf: board private structure
5979  **/
5980 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
5981 {
5982         u32 val, fcnt_prog;
5983
5984         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
5985         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
5986                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
5987                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
5988         return fcnt_prog;
5989 }
5990
5991 /**
5992  * i40e_get_global_fd_count - Get total FD filters programmed on device
5993  * @pf: board private structure
5994  **/
5995 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
5996 {
5997         u32 val, fcnt_prog;
5998
5999         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6000         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6001                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6002                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6003         return fcnt_prog;
6004 }
6005
6006 /**
6007  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6008  * @pf: board private structure
6009  **/
6010 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6011 {
6012         struct i40e_fdir_filter *filter;
6013         u32 fcnt_prog, fcnt_avail;
6014         struct hlist_node *node;
6015
6016         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
6017                 return;
6018
6019         /* Check if, FD SB or ATR was auto disabled and if there is enough room
6020          * to re-enable
6021          */
6022         fcnt_prog = i40e_get_global_fd_count(pf);
6023         fcnt_avail = pf->fdir_pf_filter_count;
6024         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6025             (pf->fd_add_err == 0) ||
6026             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6027                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6028                     (pf->hw_disabled_flags & I40E_FLAG_FD_SB_ENABLED)) {
6029                         pf->hw_disabled_flags &= ~I40E_FLAG_FD_SB_ENABLED;
6030                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
6031                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6032                 }
6033         }
6034
6035         /* Wait for some more space to be available to turn on ATR. We also
6036          * must check that no existing ntuple rules for TCP are in effect
6037          */
6038         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
6039                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6040                     (pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED) &&
6041                     (pf->fd_tcp4_filter_cnt == 0)) {
6042                         pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
6043                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
6044                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6045                 }
6046         }
6047
6048         /* if hw had a problem adding a filter, delete it */
6049         if (pf->fd_inv > 0) {
6050                 hlist_for_each_entry_safe(filter, node,
6051                                           &pf->fdir_filter_list, fdir_node) {
6052                         if (filter->fd_id == pf->fd_inv) {
6053                                 hlist_del(&filter->fdir_node);
6054                                 kfree(filter);
6055                                 pf->fdir_pf_active_filters--;
6056                         }
6057                 }
6058         }
6059 }
6060
6061 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6062 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6063 /**
6064  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6065  * @pf: board private structure
6066  **/
6067 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6068 {
6069         unsigned long min_flush_time;
6070         int flush_wait_retry = 50;
6071         bool disable_atr = false;
6072         int fd_room;
6073         int reg;
6074
6075         if (!time_after(jiffies, pf->fd_flush_timestamp +
6076                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6077                 return;
6078
6079         /* If the flush is happening too quick and we have mostly SB rules we
6080          * should not re-enable ATR for some time.
6081          */
6082         min_flush_time = pf->fd_flush_timestamp +
6083                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6084         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6085
6086         if (!(time_after(jiffies, min_flush_time)) &&
6087             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6088                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6089                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6090                 disable_atr = true;
6091         }
6092
6093         pf->fd_flush_timestamp = jiffies;
6094         pf->hw_disabled_flags |= I40E_FLAG_FD_ATR_ENABLED;
6095         /* flush all filters */
6096         wr32(&pf->hw, I40E_PFQF_CTL_1,
6097              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6098         i40e_flush(&pf->hw);
6099         pf->fd_flush_cnt++;
6100         pf->fd_add_err = 0;
6101         do {
6102                 /* Check FD flush status every 5-6msec */
6103                 usleep_range(5000, 6000);
6104                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6105                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6106                         break;
6107         } while (flush_wait_retry--);
6108         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6109                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6110         } else {
6111                 /* replay sideband filters */
6112                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6113                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
6114                         pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
6115                 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
6116                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6117                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6118         }
6119 }
6120
6121 /**
6122  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6123  * @pf: board private structure
6124  **/
6125 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6126 {
6127         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6128 }
6129
6130 /* We can see up to 256 filter programming desc in transit if the filters are
6131  * being applied really fast; before we see the first
6132  * filter miss error on Rx queue 0. Accumulating enough error messages before
6133  * reacting will make sure we don't cause flush too often.
6134  */
6135 #define I40E_MAX_FD_PROGRAM_ERROR 256
6136
6137 /**
6138  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6139  * @pf: board private structure
6140  **/
6141 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6142 {
6143
6144         /* if interface is down do nothing */
6145         if (test_bit(__I40E_DOWN, &pf->state))
6146                 return;
6147
6148         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
6149                 i40e_fdir_flush_and_replay(pf);
6150
6151         i40e_fdir_check_and_reenable(pf);
6152
6153 }
6154
6155 /**
6156  * i40e_vsi_link_event - notify VSI of a link event
6157  * @vsi: vsi to be notified
6158  * @link_up: link up or down
6159  **/
6160 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6161 {
6162         if (!vsi || test_bit(__I40E_DOWN, &vsi->state))
6163                 return;
6164
6165         switch (vsi->type) {
6166         case I40E_VSI_MAIN:
6167                 if (!vsi->netdev || !vsi->netdev_registered)
6168                         break;
6169
6170                 if (link_up) {
6171                         netif_carrier_on(vsi->netdev);
6172                         netif_tx_wake_all_queues(vsi->netdev);
6173                 } else {
6174                         netif_carrier_off(vsi->netdev);
6175                         netif_tx_stop_all_queues(vsi->netdev);
6176                 }
6177                 break;
6178
6179         case I40E_VSI_SRIOV:
6180         case I40E_VSI_VMDQ2:
6181         case I40E_VSI_CTRL:
6182         case I40E_VSI_IWARP:
6183         case I40E_VSI_MIRROR:
6184         default:
6185                 /* there is no notification for other VSIs */
6186                 break;
6187         }
6188 }
6189
6190 /**
6191  * i40e_veb_link_event - notify elements on the veb of a link event
6192  * @veb: veb to be notified
6193  * @link_up: link up or down
6194  **/
6195 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6196 {
6197         struct i40e_pf *pf;
6198         int i;
6199
6200         if (!veb || !veb->pf)
6201                 return;
6202         pf = veb->pf;
6203
6204         /* depth first... */
6205         for (i = 0; i < I40E_MAX_VEB; i++)
6206                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6207                         i40e_veb_link_event(pf->veb[i], link_up);
6208
6209         /* ... now the local VSIs */
6210         for (i = 0; i < pf->num_alloc_vsi; i++)
6211                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6212                         i40e_vsi_link_event(pf->vsi[i], link_up);
6213 }
6214
6215 /**
6216  * i40e_link_event - Update netif_carrier status
6217  * @pf: board private structure
6218  **/
6219 static void i40e_link_event(struct i40e_pf *pf)
6220 {
6221         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6222         u8 new_link_speed, old_link_speed;
6223         i40e_status status;
6224         bool new_link, old_link;
6225
6226         /* save off old link status information */
6227         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6228
6229         /* set this to force the get_link_status call to refresh state */
6230         pf->hw.phy.get_link_info = true;
6231
6232         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6233
6234         status = i40e_get_link_status(&pf->hw, &new_link);
6235
6236         /* On success, disable temp link polling */
6237         if (status == I40E_SUCCESS) {
6238                 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)
6239                         pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING;
6240         } else {
6241                 /* Enable link polling temporarily until i40e_get_link_status
6242                  * returns I40E_SUCCESS
6243                  */
6244                 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING;
6245                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6246                         status);
6247                 return;
6248         }
6249
6250         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6251         new_link_speed = pf->hw.phy.link_info.link_speed;
6252
6253         if (new_link == old_link &&
6254             new_link_speed == old_link_speed &&
6255             (test_bit(__I40E_DOWN, &vsi->state) ||
6256              new_link == netif_carrier_ok(vsi->netdev)))
6257                 return;
6258
6259         if (!test_bit(__I40E_DOWN, &vsi->state))
6260                 i40e_print_link_message(vsi, new_link);
6261
6262         /* Notify the base of the switch tree connected to
6263          * the link.  Floating VEBs are not notified.
6264          */
6265         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6266                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6267         else
6268                 i40e_vsi_link_event(vsi, new_link);
6269
6270         if (pf->vf)
6271                 i40e_vc_notify_link_state(pf);
6272
6273         if (pf->flags & I40E_FLAG_PTP)
6274                 i40e_ptp_set_increment(pf);
6275 }
6276
6277 /**
6278  * i40e_watchdog_subtask - periodic checks not using event driven response
6279  * @pf: board private structure
6280  **/
6281 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6282 {
6283         int i;
6284
6285         /* if interface is down do nothing */
6286         if (test_bit(__I40E_DOWN, &pf->state) ||
6287             test_bit(__I40E_CONFIG_BUSY, &pf->state))
6288                 return;
6289
6290         /* make sure we don't do these things too often */
6291         if (time_before(jiffies, (pf->service_timer_previous +
6292                                   pf->service_timer_period)))
6293                 return;
6294         pf->service_timer_previous = jiffies;
6295
6296         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
6297             (pf->flags & I40E_FLAG_TEMP_LINK_POLLING))
6298                 i40e_link_event(pf);
6299
6300         /* Update the stats for active netdevs so the network stack
6301          * can look at updated numbers whenever it cares to
6302          */
6303         for (i = 0; i < pf->num_alloc_vsi; i++)
6304                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6305                         i40e_update_stats(pf->vsi[i]);
6306
6307         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6308                 /* Update the stats for the active switching components */
6309                 for (i = 0; i < I40E_MAX_VEB; i++)
6310                         if (pf->veb[i])
6311                                 i40e_update_veb_stats(pf->veb[i]);
6312         }
6313
6314         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
6315 }
6316
6317 /**
6318  * i40e_reset_subtask - Set up for resetting the device and driver
6319  * @pf: board private structure
6320  **/
6321 static void i40e_reset_subtask(struct i40e_pf *pf)
6322 {
6323         u32 reset_flags = 0;
6324
6325         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
6326                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6327                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
6328         }
6329         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
6330                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6331                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
6332         }
6333         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
6334                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6335                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
6336         }
6337         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
6338                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6339                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
6340         }
6341         if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) {
6342                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6343                 clear_bit(__I40E_DOWN_REQUESTED, &pf->state);
6344         }
6345
6346         /* If there's a recovery already waiting, it takes
6347          * precedence before starting a new reset sequence.
6348          */
6349         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
6350                 i40e_prep_for_reset(pf, false);
6351                 i40e_reset(pf);
6352                 i40e_rebuild(pf, false, false);
6353         }
6354
6355         /* If we're already down or resetting, just bail */
6356         if (reset_flags &&
6357             !test_bit(__I40E_DOWN, &pf->state) &&
6358             !test_bit(__I40E_CONFIG_BUSY, &pf->state)) {
6359                 rtnl_lock();
6360                 i40e_do_reset(pf, reset_flags, true);
6361                 rtnl_unlock();
6362         }
6363 }
6364
6365 /**
6366  * i40e_handle_link_event - Handle link event
6367  * @pf: board private structure
6368  * @e: event info posted on ARQ
6369  **/
6370 static void i40e_handle_link_event(struct i40e_pf *pf,
6371                                    struct i40e_arq_event_info *e)
6372 {
6373         struct i40e_aqc_get_link_status *status =
6374                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6375
6376         /* Do a new status request to re-enable LSE reporting
6377          * and load new status information into the hw struct
6378          * This completely ignores any state information
6379          * in the ARQ event info, instead choosing to always
6380          * issue the AQ update link status command.
6381          */
6382         i40e_link_event(pf);
6383
6384         /* check for unqualified module, if link is down */
6385         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6386             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6387             (!(status->link_info & I40E_AQ_LINK_UP)))
6388                 dev_err(&pf->pdev->dev,
6389                         "The driver failed to link because an unqualified module was detected.\n");
6390 }
6391
6392 /**
6393  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6394  * @pf: board private structure
6395  **/
6396 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6397 {
6398         struct i40e_arq_event_info event;
6399         struct i40e_hw *hw = &pf->hw;
6400         u16 pending, i = 0;
6401         i40e_status ret;
6402         u16 opcode;
6403         u32 oldval;
6404         u32 val;
6405
6406         /* Do not run clean AQ when PF reset fails */
6407         if (test_bit(__I40E_RESET_FAILED, &pf->state))
6408                 return;
6409
6410         /* check for error indications */
6411         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6412         oldval = val;
6413         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6414                 if (hw->debug_mask & I40E_DEBUG_AQ)
6415                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6416                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6417         }
6418         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6419                 if (hw->debug_mask & I40E_DEBUG_AQ)
6420                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6421                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6422                 pf->arq_overflows++;
6423         }
6424         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6425                 if (hw->debug_mask & I40E_DEBUG_AQ)
6426                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6427                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6428         }
6429         if (oldval != val)
6430                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6431
6432         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6433         oldval = val;
6434         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6435                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6436                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6437                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6438         }
6439         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6440                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6441                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6442                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6443         }
6444         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6445                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6446                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6447                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6448         }
6449         if (oldval != val)
6450                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6451
6452         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6453         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6454         if (!event.msg_buf)
6455                 return;
6456
6457         do {
6458                 ret = i40e_clean_arq_element(hw, &event, &pending);
6459                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6460                         break;
6461                 else if (ret) {
6462                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6463                         break;
6464                 }
6465
6466                 opcode = le16_to_cpu(event.desc.opcode);
6467                 switch (opcode) {
6468
6469                 case i40e_aqc_opc_get_link_status:
6470                         i40e_handle_link_event(pf, &event);
6471                         break;
6472                 case i40e_aqc_opc_send_msg_to_pf:
6473                         ret = i40e_vc_process_vf_msg(pf,
6474                                         le16_to_cpu(event.desc.retval),
6475                                         le32_to_cpu(event.desc.cookie_high),
6476                                         le32_to_cpu(event.desc.cookie_low),
6477                                         event.msg_buf,
6478                                         event.msg_len);
6479                         break;
6480                 case i40e_aqc_opc_lldp_update_mib:
6481                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6482 #ifdef CONFIG_I40E_DCB
6483                         rtnl_lock();
6484                         ret = i40e_handle_lldp_event(pf, &event);
6485                         rtnl_unlock();
6486 #endif /* CONFIG_I40E_DCB */
6487                         break;
6488                 case i40e_aqc_opc_event_lan_overflow:
6489                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6490                         i40e_handle_lan_overflow_event(pf, &event);
6491                         break;
6492                 case i40e_aqc_opc_send_msg_to_peer:
6493                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6494                         break;
6495                 case i40e_aqc_opc_nvm_erase:
6496                 case i40e_aqc_opc_nvm_update:
6497                 case i40e_aqc_opc_oem_post_update:
6498                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6499                                    "ARQ NVM operation 0x%04x completed\n",
6500                                    opcode);
6501                         break;
6502                 default:
6503                         dev_info(&pf->pdev->dev,
6504                                  "ARQ: Unknown event 0x%04x ignored\n",
6505                                  opcode);
6506                         break;
6507                 }
6508         } while (i++ < pf->adminq_work_limit);
6509
6510         if (i < pf->adminq_work_limit)
6511                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
6512
6513         /* re-enable Admin queue interrupt cause */
6514         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6515         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6516         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6517         i40e_flush(hw);
6518
6519         kfree(event.msg_buf);
6520 }
6521
6522 /**
6523  * i40e_verify_eeprom - make sure eeprom is good to use
6524  * @pf: board private structure
6525  **/
6526 static void i40e_verify_eeprom(struct i40e_pf *pf)
6527 {
6528         int err;
6529
6530         err = i40e_diag_eeprom_test(&pf->hw);
6531         if (err) {
6532                 /* retry in case of garbage read */
6533                 err = i40e_diag_eeprom_test(&pf->hw);
6534                 if (err) {
6535                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6536                                  err);
6537                         set_bit(__I40E_BAD_EEPROM, &pf->state);
6538                 }
6539         }
6540
6541         if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
6542                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6543                 clear_bit(__I40E_BAD_EEPROM, &pf->state);
6544         }
6545 }
6546
6547 /**
6548  * i40e_enable_pf_switch_lb
6549  * @pf: pointer to the PF structure
6550  *
6551  * enable switch loop back or die - no point in a return value
6552  **/
6553 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6554 {
6555         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6556         struct i40e_vsi_context ctxt;
6557         int ret;
6558
6559         ctxt.seid = pf->main_vsi_seid;
6560         ctxt.pf_num = pf->hw.pf_id;
6561         ctxt.vf_num = 0;
6562         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6563         if (ret) {
6564                 dev_info(&pf->pdev->dev,
6565                          "couldn't get PF vsi config, err %s aq_err %s\n",
6566                          i40e_stat_str(&pf->hw, ret),
6567                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6568                 return;
6569         }
6570         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6571         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6572         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6573
6574         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6575         if (ret) {
6576                 dev_info(&pf->pdev->dev,
6577                          "update vsi switch failed, err %s aq_err %s\n",
6578                          i40e_stat_str(&pf->hw, ret),
6579                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6580         }
6581 }
6582
6583 /**
6584  * i40e_disable_pf_switch_lb
6585  * @pf: pointer to the PF structure
6586  *
6587  * disable switch loop back or die - no point in a return value
6588  **/
6589 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6590 {
6591         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6592         struct i40e_vsi_context ctxt;
6593         int ret;
6594
6595         ctxt.seid = pf->main_vsi_seid;
6596         ctxt.pf_num = pf->hw.pf_id;
6597         ctxt.vf_num = 0;
6598         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6599         if (ret) {
6600                 dev_info(&pf->pdev->dev,
6601                          "couldn't get PF vsi config, err %s aq_err %s\n",
6602                          i40e_stat_str(&pf->hw, ret),
6603                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6604                 return;
6605         }
6606         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6607         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6608         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6609
6610         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6611         if (ret) {
6612                 dev_info(&pf->pdev->dev,
6613                          "update vsi switch failed, err %s aq_err %s\n",
6614                          i40e_stat_str(&pf->hw, ret),
6615                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6616         }
6617 }
6618
6619 /**
6620  * i40e_config_bridge_mode - Configure the HW bridge mode
6621  * @veb: pointer to the bridge instance
6622  *
6623  * Configure the loop back mode for the LAN VSI that is downlink to the
6624  * specified HW bridge instance. It is expected this function is called
6625  * when a new HW bridge is instantiated.
6626  **/
6627 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6628 {
6629         struct i40e_pf *pf = veb->pf;
6630
6631         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6632                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6633                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6634         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6635                 i40e_disable_pf_switch_lb(pf);
6636         else
6637                 i40e_enable_pf_switch_lb(pf);
6638 }
6639
6640 /**
6641  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6642  * @veb: pointer to the VEB instance
6643  *
6644  * This is a recursive function that first builds the attached VSIs then
6645  * recurses in to build the next layer of VEB.  We track the connections
6646  * through our own index numbers because the seid's from the HW could
6647  * change across the reset.
6648  **/
6649 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6650 {
6651         struct i40e_vsi *ctl_vsi = NULL;
6652         struct i40e_pf *pf = veb->pf;
6653         int v, veb_idx;
6654         int ret;
6655
6656         /* build VSI that owns this VEB, temporarily attached to base VEB */
6657         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6658                 if (pf->vsi[v] &&
6659                     pf->vsi[v]->veb_idx == veb->idx &&
6660                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6661                         ctl_vsi = pf->vsi[v];
6662                         break;
6663                 }
6664         }
6665         if (!ctl_vsi) {
6666                 dev_info(&pf->pdev->dev,
6667                          "missing owner VSI for veb_idx %d\n", veb->idx);
6668                 ret = -ENOENT;
6669                 goto end_reconstitute;
6670         }
6671         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6672                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6673         ret = i40e_add_vsi(ctl_vsi);
6674         if (ret) {
6675                 dev_info(&pf->pdev->dev,
6676                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6677                          veb->idx, ret);
6678                 goto end_reconstitute;
6679         }
6680         i40e_vsi_reset_stats(ctl_vsi);
6681
6682         /* create the VEB in the switch and move the VSI onto the VEB */
6683         ret = i40e_add_veb(veb, ctl_vsi);
6684         if (ret)
6685                 goto end_reconstitute;
6686
6687         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6688                 veb->bridge_mode = BRIDGE_MODE_VEB;
6689         else
6690                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6691         i40e_config_bridge_mode(veb);
6692
6693         /* create the remaining VSIs attached to this VEB */
6694         for (v = 0; v < pf->num_alloc_vsi; v++) {
6695                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6696                         continue;
6697
6698                 if (pf->vsi[v]->veb_idx == veb->idx) {
6699                         struct i40e_vsi *vsi = pf->vsi[v];
6700
6701                         vsi->uplink_seid = veb->seid;
6702                         ret = i40e_add_vsi(vsi);
6703                         if (ret) {
6704                                 dev_info(&pf->pdev->dev,
6705                                          "rebuild of vsi_idx %d failed: %d\n",
6706                                          v, ret);
6707                                 goto end_reconstitute;
6708                         }
6709                         i40e_vsi_reset_stats(vsi);
6710                 }
6711         }
6712
6713         /* create any VEBs attached to this VEB - RECURSION */
6714         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6715                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6716                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6717                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6718                         if (ret)
6719                                 break;
6720                 }
6721         }
6722
6723 end_reconstitute:
6724         return ret;
6725 }
6726
6727 /**
6728  * i40e_get_capabilities - get info about the HW
6729  * @pf: the PF struct
6730  **/
6731 static int i40e_get_capabilities(struct i40e_pf *pf)
6732 {
6733         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6734         u16 data_size;
6735         int buf_len;
6736         int err;
6737
6738         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6739         do {
6740                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6741                 if (!cap_buf)
6742                         return -ENOMEM;
6743
6744                 /* this loads the data into the hw struct for us */
6745                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6746                                             &data_size,
6747                                             i40e_aqc_opc_list_func_capabilities,
6748                                             NULL);
6749                 /* data loaded, buffer no longer needed */
6750                 kfree(cap_buf);
6751
6752                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6753                         /* retry with a larger buffer */
6754                         buf_len = data_size;
6755                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6756                         dev_info(&pf->pdev->dev,
6757                                  "capability discovery failed, err %s aq_err %s\n",
6758                                  i40e_stat_str(&pf->hw, err),
6759                                  i40e_aq_str(&pf->hw,
6760                                              pf->hw.aq.asq_last_status));
6761                         return -ENODEV;
6762                 }
6763         } while (err);
6764
6765         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6766                 dev_info(&pf->pdev->dev,
6767                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6768                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6769                          pf->hw.func_caps.num_msix_vectors,
6770                          pf->hw.func_caps.num_msix_vectors_vf,
6771                          pf->hw.func_caps.fd_filters_guaranteed,
6772                          pf->hw.func_caps.fd_filters_best_effort,
6773                          pf->hw.func_caps.num_tx_qp,
6774                          pf->hw.func_caps.num_vsis);
6775
6776 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6777                        + pf->hw.func_caps.num_vfs)
6778         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6779                 dev_info(&pf->pdev->dev,
6780                          "got num_vsis %d, setting num_vsis to %d\n",
6781                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6782                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6783         }
6784
6785         return 0;
6786 }
6787
6788 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6789
6790 /**
6791  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6792  * @pf: board private structure
6793  **/
6794 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6795 {
6796         struct i40e_vsi *vsi;
6797
6798         /* quick workaround for an NVM issue that leaves a critical register
6799          * uninitialized
6800          */
6801         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6802                 static const u32 hkey[] = {
6803                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6804                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6805                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6806                         0x95b3a76d};
6807                 int i;
6808
6809                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
6810                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
6811         }
6812
6813         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6814                 return;
6815
6816         /* find existing VSI and see if it needs configuring */
6817         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6818
6819         /* create a new VSI if none exists */
6820         if (!vsi) {
6821                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
6822                                      pf->vsi[pf->lan_vsi]->seid, 0);
6823                 if (!vsi) {
6824                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
6825                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
6826                         return;
6827                 }
6828         }
6829
6830         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
6831 }
6832
6833 /**
6834  * i40e_fdir_teardown - release the Flow Director resources
6835  * @pf: board private structure
6836  **/
6837 static void i40e_fdir_teardown(struct i40e_pf *pf)
6838 {
6839         struct i40e_vsi *vsi;
6840
6841         i40e_fdir_filter_exit(pf);
6842         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6843         if (vsi)
6844                 i40e_vsi_release(vsi);
6845 }
6846
6847 /**
6848  * i40e_prep_for_reset - prep for the core to reset
6849  * @pf: board private structure
6850  * @lock_acquired: indicates whether or not the lock has been acquired
6851  * before this function was called.
6852  *
6853  * Close up the VFs and other things in prep for PF Reset.
6854   **/
6855 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
6856 {
6857         struct i40e_hw *hw = &pf->hw;
6858         i40e_status ret = 0;
6859         u32 v;
6860
6861         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
6862         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
6863                 return;
6864         if (i40e_check_asq_alive(&pf->hw))
6865                 i40e_vc_notify_reset(pf);
6866
6867         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
6868
6869         /* quiesce the VSIs and their queues that are not already DOWN */
6870         /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */
6871         if (!lock_acquired)
6872                 rtnl_lock();
6873         i40e_pf_quiesce_all_vsi(pf);
6874         if (!lock_acquired)
6875                 rtnl_unlock();
6876
6877         for (v = 0; v < pf->num_alloc_vsi; v++) {
6878                 if (pf->vsi[v])
6879                         pf->vsi[v]->seid = 0;
6880         }
6881
6882         i40e_shutdown_adminq(&pf->hw);
6883
6884         /* call shutdown HMC */
6885         if (hw->hmc.hmc_obj) {
6886                 ret = i40e_shutdown_lan_hmc(hw);
6887                 if (ret)
6888                         dev_warn(&pf->pdev->dev,
6889                                  "shutdown_lan_hmc failed: %d\n", ret);
6890         }
6891 }
6892
6893 /**
6894  * i40e_send_version - update firmware with driver version
6895  * @pf: PF struct
6896  */
6897 static void i40e_send_version(struct i40e_pf *pf)
6898 {
6899         struct i40e_driver_version dv;
6900
6901         dv.major_version = DRV_VERSION_MAJOR;
6902         dv.minor_version = DRV_VERSION_MINOR;
6903         dv.build_version = DRV_VERSION_BUILD;
6904         dv.subbuild_version = 0;
6905         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
6906         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
6907 }
6908
6909 /**
6910  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
6911  * @pf: board private structure
6912  **/
6913 static int i40e_reset(struct i40e_pf *pf)
6914 {
6915         struct i40e_hw *hw = &pf->hw;
6916         i40e_status ret;
6917
6918         ret = i40e_pf_reset(hw);
6919         if (ret) {
6920                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
6921                 set_bit(__I40E_RESET_FAILED, &pf->state);
6922                 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
6923         } else {
6924                 pf->pfr_count++;
6925         }
6926         return ret;
6927 }
6928
6929 /**
6930  * i40e_rebuild - rebuild using a saved config
6931  * @pf: board private structure
6932  * @reinit: if the Main VSI needs to re-initialized.
6933  * @lock_acquired: indicates whether or not the lock has been acquired
6934  * before this function was called.
6935  **/
6936 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
6937 {
6938         struct i40e_hw *hw = &pf->hw;
6939         u8 set_fc_aq_fail = 0;
6940         i40e_status ret;
6941         u32 val;
6942         int v;
6943
6944         if (test_bit(__I40E_DOWN, &pf->state))
6945                 goto clear_recovery;
6946         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
6947
6948         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
6949         ret = i40e_init_adminq(&pf->hw);
6950         if (ret) {
6951                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
6952                          i40e_stat_str(&pf->hw, ret),
6953                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6954                 goto clear_recovery;
6955         }
6956
6957         /* re-verify the eeprom if we just had an EMP reset */
6958         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state))
6959                 i40e_verify_eeprom(pf);
6960
6961         i40e_clear_pxe_mode(hw);
6962         ret = i40e_get_capabilities(pf);
6963         if (ret)
6964                 goto end_core_reset;
6965
6966         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
6967                                 hw->func_caps.num_rx_qp, 0, 0);
6968         if (ret) {
6969                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
6970                 goto end_core_reset;
6971         }
6972         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
6973         if (ret) {
6974                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
6975                 goto end_core_reset;
6976         }
6977
6978 #ifdef CONFIG_I40E_DCB
6979         ret = i40e_init_pf_dcb(pf);
6980         if (ret) {
6981                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
6982                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
6983                 /* Continue without DCB enabled */
6984         }
6985 #endif /* CONFIG_I40E_DCB */
6986         /* do basic switch setup */
6987         if (!lock_acquired)
6988                 rtnl_lock();
6989         ret = i40e_setup_pf_switch(pf, reinit);
6990         if (ret)
6991                 goto end_unlock;
6992
6993         /* The driver only wants link up/down and module qualification
6994          * reports from firmware.  Note the negative logic.
6995          */
6996         ret = i40e_aq_set_phy_int_mask(&pf->hw,
6997                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
6998                                          I40E_AQ_EVENT_MEDIA_NA |
6999                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7000         if (ret)
7001                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7002                          i40e_stat_str(&pf->hw, ret),
7003                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7004
7005         /* make sure our flow control settings are restored */
7006         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7007         if (ret)
7008                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7009                         i40e_stat_str(&pf->hw, ret),
7010                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7011
7012         /* Rebuild the VSIs and VEBs that existed before reset.
7013          * They are still in our local switch element arrays, so only
7014          * need to rebuild the switch model in the HW.
7015          *
7016          * If there were VEBs but the reconstitution failed, we'll try
7017          * try to recover minimal use by getting the basic PF VSI working.
7018          */
7019         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7020                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7021                 /* find the one VEB connected to the MAC, and find orphans */
7022                 for (v = 0; v < I40E_MAX_VEB; v++) {
7023                         if (!pf->veb[v])
7024                                 continue;
7025
7026                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7027                             pf->veb[v]->uplink_seid == 0) {
7028                                 ret = i40e_reconstitute_veb(pf->veb[v]);
7029
7030                                 if (!ret)
7031                                         continue;
7032
7033                                 /* If Main VEB failed, we're in deep doodoo,
7034                                  * so give up rebuilding the switch and set up
7035                                  * for minimal rebuild of PF VSI.
7036                                  * If orphan failed, we'll report the error
7037                                  * but try to keep going.
7038                                  */
7039                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7040                                         dev_info(&pf->pdev->dev,
7041                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7042                                                  ret);
7043                                         pf->vsi[pf->lan_vsi]->uplink_seid
7044                                                                 = pf->mac_seid;
7045                                         break;
7046                                 } else if (pf->veb[v]->uplink_seid == 0) {
7047                                         dev_info(&pf->pdev->dev,
7048                                                  "rebuild of orphan VEB failed: %d\n",
7049                                                  ret);
7050                                 }
7051                         }
7052                 }
7053         }
7054
7055         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7056                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7057                 /* no VEB, so rebuild only the Main VSI */
7058                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7059                 if (ret) {
7060                         dev_info(&pf->pdev->dev,
7061                                  "rebuild of Main VSI failed: %d\n", ret);
7062                         goto end_unlock;
7063                 }
7064         }
7065
7066         /* Reconfigure hardware for allowing smaller MSS in the case
7067          * of TSO, so that we avoid the MDD being fired and causing
7068          * a reset in the case of small MSS+TSO.
7069          */
7070 #define I40E_REG_MSS          0x000E64DC
7071 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7072 #define I40E_64BYTE_MSS       0x400000
7073         val = rd32(hw, I40E_REG_MSS);
7074         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7075                 val &= ~I40E_REG_MSS_MIN_MASK;
7076                 val |= I40E_64BYTE_MSS;
7077                 wr32(hw, I40E_REG_MSS, val);
7078         }
7079
7080         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
7081                 msleep(75);
7082                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7083                 if (ret)
7084                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7085                                  i40e_stat_str(&pf->hw, ret),
7086                                  i40e_aq_str(&pf->hw,
7087                                              pf->hw.aq.asq_last_status));
7088         }
7089         /* reinit the misc interrupt */
7090         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7091                 ret = i40e_setup_misc_vector(pf);
7092
7093         /* Add a filter to drop all Flow control frames from any VSI from being
7094          * transmitted. By doing so we stop a malicious VF from sending out
7095          * PAUSE or PFC frames and potentially controlling traffic for other
7096          * PF/VF VSIs.
7097          * The FW can still send Flow control frames if enabled.
7098          */
7099         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7100                                                        pf->main_vsi_seid);
7101
7102         /* restart the VSIs that were rebuilt and running before the reset */
7103         i40e_pf_unquiesce_all_vsi(pf);
7104
7105         if (pf->num_alloc_vfs) {
7106                 for (v = 0; v < pf->num_alloc_vfs; v++)
7107                         i40e_reset_vf(&pf->vf[v], true);
7108         }
7109
7110         /* tell the firmware that we're starting */
7111         i40e_send_version(pf);
7112
7113 end_unlock:
7114 if (!lock_acquired)
7115         rtnl_unlock();
7116 end_core_reset:
7117         clear_bit(__I40E_RESET_FAILED, &pf->state);
7118 clear_recovery:
7119         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
7120 }
7121
7122 /**
7123  * i40e_reset_and_rebuild - reset and rebuild using a saved config
7124  * @pf: board private structure
7125  * @reinit: if the Main VSI needs to re-initialized.
7126  * @lock_acquired: indicates whether or not the lock has been acquired
7127  * before this function was called.
7128  **/
7129 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
7130                                    bool lock_acquired)
7131 {
7132         int ret;
7133         /* Now we wait for GRST to settle out.
7134          * We don't have to delete the VEBs or VSIs from the hw switch
7135          * because the reset will make them disappear.
7136          */
7137         ret = i40e_reset(pf);
7138         if (!ret)
7139                 i40e_rebuild(pf, reinit, lock_acquired);
7140 }
7141
7142 /**
7143  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7144  * @pf: board private structure
7145  *
7146  * Close up the VFs and other things in prep for a Core Reset,
7147  * then get ready to rebuild the world.
7148  * @lock_acquired: indicates whether or not the lock has been acquired
7149  * before this function was called.
7150  **/
7151 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
7152 {
7153         i40e_prep_for_reset(pf, lock_acquired);
7154         i40e_reset_and_rebuild(pf, false, lock_acquired);
7155 }
7156
7157 /**
7158  * i40e_handle_mdd_event
7159  * @pf: pointer to the PF structure
7160  *
7161  * Called from the MDD irq handler to identify possibly malicious vfs
7162  **/
7163 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7164 {
7165         struct i40e_hw *hw = &pf->hw;
7166         bool mdd_detected = false;
7167         bool pf_mdd_detected = false;
7168         struct i40e_vf *vf;
7169         u32 reg;
7170         int i;
7171
7172         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
7173                 return;
7174
7175         /* find what triggered the MDD event */
7176         reg = rd32(hw, I40E_GL_MDET_TX);
7177         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7178                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7179                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7180                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7181                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7182                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7183                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7184                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7185                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7186                                 pf->hw.func_caps.base_queue;
7187                 if (netif_msg_tx_err(pf))
7188                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7189                                  event, queue, pf_num, vf_num);
7190                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7191                 mdd_detected = true;
7192         }
7193         reg = rd32(hw, I40E_GL_MDET_RX);
7194         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7195                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7196                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7197                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7198                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7199                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7200                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7201                                 pf->hw.func_caps.base_queue;
7202                 if (netif_msg_rx_err(pf))
7203                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7204                                  event, queue, func);
7205                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7206                 mdd_detected = true;
7207         }
7208
7209         if (mdd_detected) {
7210                 reg = rd32(hw, I40E_PF_MDET_TX);
7211                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7212                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7213                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7214                         pf_mdd_detected = true;
7215                 }
7216                 reg = rd32(hw, I40E_PF_MDET_RX);
7217                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7218                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7219                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7220                         pf_mdd_detected = true;
7221                 }
7222                 /* Queue belongs to the PF, initiate a reset */
7223                 if (pf_mdd_detected) {
7224                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
7225                         i40e_service_event_schedule(pf);
7226                 }
7227         }
7228
7229         /* see if one of the VFs needs its hand slapped */
7230         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7231                 vf = &(pf->vf[i]);
7232                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7233                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7234                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7235                         vf->num_mdd_events++;
7236                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7237                                  i);
7238                 }
7239
7240                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7241                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7242                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7243                         vf->num_mdd_events++;
7244                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7245                                  i);
7246                 }
7247
7248                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7249                         dev_info(&pf->pdev->dev,
7250                                  "Too many MDD events on VF %d, disabled\n", i);
7251                         dev_info(&pf->pdev->dev,
7252                                  "Use PF Control I/F to re-enable the VF\n");
7253                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
7254                 }
7255         }
7256
7257         /* re-enable mdd interrupt cause */
7258         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
7259         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7260         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7261         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7262         i40e_flush(hw);
7263 }
7264
7265 /**
7266  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7267  * @pf: board private structure
7268  **/
7269 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7270 {
7271         struct i40e_hw *hw = &pf->hw;
7272         i40e_status ret;
7273         u16 port;
7274         int i;
7275
7276         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7277                 return;
7278
7279         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7280
7281         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7282                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7283                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7284                         port = pf->udp_ports[i].index;
7285                         if (port)
7286                                 ret = i40e_aq_add_udp_tunnel(hw, port,
7287                                                         pf->udp_ports[i].type,
7288                                                         NULL, NULL);
7289                         else
7290                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7291
7292                         if (ret) {
7293                                 dev_dbg(&pf->pdev->dev,
7294                                         "%s %s port %d, index %d failed, err %s aq_err %s\n",
7295                                         pf->udp_ports[i].type ? "vxlan" : "geneve",
7296                                         port ? "add" : "delete",
7297                                         port, i,
7298                                         i40e_stat_str(&pf->hw, ret),
7299                                         i40e_aq_str(&pf->hw,
7300                                                     pf->hw.aq.asq_last_status));
7301                                 pf->udp_ports[i].index = 0;
7302                         }
7303                 }
7304         }
7305 }
7306
7307 /**
7308  * i40e_service_task - Run the driver's async subtasks
7309  * @work: pointer to work_struct containing our data
7310  **/
7311 static void i40e_service_task(struct work_struct *work)
7312 {
7313         struct i40e_pf *pf = container_of(work,
7314                                           struct i40e_pf,
7315                                           service_task);
7316         unsigned long start_time = jiffies;
7317
7318         /* don't bother with service tasks if a reset is in progress */
7319         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
7320                 return;
7321         }
7322
7323         if (test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
7324                 return;
7325
7326         i40e_detect_recover_hung(pf);
7327         i40e_sync_filters_subtask(pf);
7328         i40e_reset_subtask(pf);
7329         i40e_handle_mdd_event(pf);
7330         i40e_vc_process_vflr_event(pf);
7331         i40e_watchdog_subtask(pf);
7332         i40e_fdir_reinit_subtask(pf);
7333         if (pf->flags & I40E_FLAG_CLIENT_RESET) {
7334                 /* Client subtask will reopen next time through. */
7335                 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true);
7336                 pf->flags &= ~I40E_FLAG_CLIENT_RESET;
7337         } else {
7338                 i40e_client_subtask(pf);
7339                 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) {
7340                         i40e_notify_client_of_l2_param_changes(
7341                                                         pf->vsi[pf->lan_vsi]);
7342                         pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE;
7343                 }
7344         }
7345         i40e_sync_filters_subtask(pf);
7346         i40e_sync_udp_filters_subtask(pf);
7347         i40e_clean_adminq_subtask(pf);
7348
7349         /* flush memory to make sure state is correct before next watchdog */
7350         smp_mb__before_atomic();
7351         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
7352
7353         /* If the tasks have taken longer than one timer cycle or there
7354          * is more work to be done, reschedule the service task now
7355          * rather than wait for the timer to tick again.
7356          */
7357         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7358             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
7359             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
7360             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
7361                 i40e_service_event_schedule(pf);
7362 }
7363
7364 /**
7365  * i40e_service_timer - timer callback
7366  * @data: pointer to PF struct
7367  **/
7368 static void i40e_service_timer(unsigned long data)
7369 {
7370         struct i40e_pf *pf = (struct i40e_pf *)data;
7371
7372         mod_timer(&pf->service_timer,
7373                   round_jiffies(jiffies + pf->service_timer_period));
7374         i40e_service_event_schedule(pf);
7375 }
7376
7377 /**
7378  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7379  * @vsi: the VSI being configured
7380  **/
7381 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7382 {
7383         struct i40e_pf *pf = vsi->back;
7384
7385         switch (vsi->type) {
7386         case I40E_VSI_MAIN:
7387                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7388                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7389                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7390                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7391                         vsi->num_q_vectors = pf->num_lan_msix;
7392                 else
7393                         vsi->num_q_vectors = 1;
7394
7395                 break;
7396
7397         case I40E_VSI_FDIR:
7398                 vsi->alloc_queue_pairs = 1;
7399                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7400                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7401                 vsi->num_q_vectors = pf->num_fdsb_msix;
7402                 break;
7403
7404         case I40E_VSI_VMDQ2:
7405                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7406                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7407                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7408                 vsi->num_q_vectors = pf->num_vmdq_msix;
7409                 break;
7410
7411         case I40E_VSI_SRIOV:
7412                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7413                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7414                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7415                 break;
7416
7417         default:
7418                 WARN_ON(1);
7419                 return -ENODATA;
7420         }
7421
7422         return 0;
7423 }
7424
7425 /**
7426  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7427  * @type: VSI pointer
7428  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7429  *
7430  * On error: returns error code (negative)
7431  * On success: returns 0
7432  **/
7433 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7434 {
7435         int size;
7436         int ret = 0;
7437
7438         /* allocate memory for both Tx and Rx ring pointers */
7439         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
7440         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7441         if (!vsi->tx_rings)
7442                 return -ENOMEM;
7443         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
7444
7445         if (alloc_qvectors) {
7446                 /* allocate memory for q_vector pointers */
7447                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7448                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7449                 if (!vsi->q_vectors) {
7450                         ret = -ENOMEM;
7451                         goto err_vectors;
7452                 }
7453         }
7454         return ret;
7455
7456 err_vectors:
7457         kfree(vsi->tx_rings);
7458         return ret;
7459 }
7460
7461 /**
7462  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7463  * @pf: board private structure
7464  * @type: type of VSI
7465  *
7466  * On error: returns error code (negative)
7467  * On success: returns vsi index in PF (positive)
7468  **/
7469 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7470 {
7471         int ret = -ENODEV;
7472         struct i40e_vsi *vsi;
7473         int vsi_idx;
7474         int i;
7475
7476         /* Need to protect the allocation of the VSIs at the PF level */
7477         mutex_lock(&pf->switch_mutex);
7478
7479         /* VSI list may be fragmented if VSI creation/destruction has
7480          * been happening.  We can afford to do a quick scan to look
7481          * for any free VSIs in the list.
7482          *
7483          * find next empty vsi slot, looping back around if necessary
7484          */
7485         i = pf->next_vsi;
7486         while (i < pf->num_alloc_vsi && pf->vsi[i])
7487                 i++;
7488         if (i >= pf->num_alloc_vsi) {
7489                 i = 0;
7490                 while (i < pf->next_vsi && pf->vsi[i])
7491                         i++;
7492         }
7493
7494         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7495                 vsi_idx = i;             /* Found one! */
7496         } else {
7497                 ret = -ENODEV;
7498                 goto unlock_pf;  /* out of VSI slots! */
7499         }
7500         pf->next_vsi = ++i;
7501
7502         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7503         if (!vsi) {
7504                 ret = -ENOMEM;
7505                 goto unlock_pf;
7506         }
7507         vsi->type = type;
7508         vsi->back = pf;
7509         set_bit(__I40E_DOWN, &vsi->state);
7510         vsi->flags = 0;
7511         vsi->idx = vsi_idx;
7512         vsi->int_rate_limit = 0;
7513         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7514                                 pf->rss_table_size : 64;
7515         vsi->netdev_registered = false;
7516         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7517         hash_init(vsi->mac_filter_hash);
7518         vsi->irqs_ready = false;
7519
7520         ret = i40e_set_num_rings_in_vsi(vsi);
7521         if (ret)
7522                 goto err_rings;
7523
7524         ret = i40e_vsi_alloc_arrays(vsi, true);
7525         if (ret)
7526                 goto err_rings;
7527
7528         /* Setup default MSIX irq handler for VSI */
7529         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7530
7531         /* Initialize VSI lock */
7532         spin_lock_init(&vsi->mac_filter_hash_lock);
7533         pf->vsi[vsi_idx] = vsi;
7534         ret = vsi_idx;
7535         goto unlock_pf;
7536
7537 err_rings:
7538         pf->next_vsi = i - 1;
7539         kfree(vsi);
7540 unlock_pf:
7541         mutex_unlock(&pf->switch_mutex);
7542         return ret;
7543 }
7544
7545 /**
7546  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7547  * @type: VSI pointer
7548  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7549  *
7550  * On error: returns error code (negative)
7551  * On success: returns 0
7552  **/
7553 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7554 {
7555         /* free the ring and vector containers */
7556         if (free_qvectors) {
7557                 kfree(vsi->q_vectors);
7558                 vsi->q_vectors = NULL;
7559         }
7560         kfree(vsi->tx_rings);
7561         vsi->tx_rings = NULL;
7562         vsi->rx_rings = NULL;
7563 }
7564
7565 /**
7566  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7567  * and lookup table
7568  * @vsi: Pointer to VSI structure
7569  */
7570 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7571 {
7572         if (!vsi)
7573                 return;
7574
7575         kfree(vsi->rss_hkey_user);
7576         vsi->rss_hkey_user = NULL;
7577
7578         kfree(vsi->rss_lut_user);
7579         vsi->rss_lut_user = NULL;
7580 }
7581
7582 /**
7583  * i40e_vsi_clear - Deallocate the VSI provided
7584  * @vsi: the VSI being un-configured
7585  **/
7586 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7587 {
7588         struct i40e_pf *pf;
7589
7590         if (!vsi)
7591                 return 0;
7592
7593         if (!vsi->back)
7594                 goto free_vsi;
7595         pf = vsi->back;
7596
7597         mutex_lock(&pf->switch_mutex);
7598         if (!pf->vsi[vsi->idx]) {
7599                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7600                         vsi->idx, vsi->idx, vsi, vsi->type);
7601                 goto unlock_vsi;
7602         }
7603
7604         if (pf->vsi[vsi->idx] != vsi) {
7605                 dev_err(&pf->pdev->dev,
7606                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7607                         pf->vsi[vsi->idx]->idx,
7608                         pf->vsi[vsi->idx],
7609                         pf->vsi[vsi->idx]->type,
7610                         vsi->idx, vsi, vsi->type);
7611                 goto unlock_vsi;
7612         }
7613
7614         /* updates the PF for this cleared vsi */
7615         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7616         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7617
7618         i40e_vsi_free_arrays(vsi, true);
7619         i40e_clear_rss_config_user(vsi);
7620
7621         pf->vsi[vsi->idx] = NULL;
7622         if (vsi->idx < pf->next_vsi)
7623                 pf->next_vsi = vsi->idx;
7624
7625 unlock_vsi:
7626         mutex_unlock(&pf->switch_mutex);
7627 free_vsi:
7628         kfree(vsi);
7629
7630         return 0;
7631 }
7632
7633 /**
7634  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7635  * @vsi: the VSI being cleaned
7636  **/
7637 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7638 {
7639         int i;
7640
7641         if (vsi->tx_rings && vsi->tx_rings[0]) {
7642                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7643                         kfree_rcu(vsi->tx_rings[i], rcu);
7644                         vsi->tx_rings[i] = NULL;
7645                         vsi->rx_rings[i] = NULL;
7646                 }
7647         }
7648 }
7649
7650 /**
7651  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7652  * @vsi: the VSI being configured
7653  **/
7654 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7655 {
7656         struct i40e_ring *tx_ring, *rx_ring;
7657         struct i40e_pf *pf = vsi->back;
7658         int i;
7659
7660         /* Set basic values in the rings to be used later during open() */
7661         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7662                 /* allocate space for both Tx and Rx in one shot */
7663                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
7664                 if (!tx_ring)
7665                         goto err_out;
7666
7667                 tx_ring->queue_index = i;
7668                 tx_ring->reg_idx = vsi->base_queue + i;
7669                 tx_ring->ring_active = false;
7670                 tx_ring->vsi = vsi;
7671                 tx_ring->netdev = vsi->netdev;
7672                 tx_ring->dev = &pf->pdev->dev;
7673                 tx_ring->count = vsi->num_desc;
7674                 tx_ring->size = 0;
7675                 tx_ring->dcb_tc = 0;
7676                 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
7677                         tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7678                 tx_ring->tx_itr_setting = pf->tx_itr_default;
7679                 vsi->tx_rings[i] = tx_ring;
7680
7681                 rx_ring = &tx_ring[1];
7682                 rx_ring->queue_index = i;
7683                 rx_ring->reg_idx = vsi->base_queue + i;
7684                 rx_ring->ring_active = false;
7685                 rx_ring->vsi = vsi;
7686                 rx_ring->netdev = vsi->netdev;
7687                 rx_ring->dev = &pf->pdev->dev;
7688                 rx_ring->count = vsi->num_desc;
7689                 rx_ring->size = 0;
7690                 rx_ring->dcb_tc = 0;
7691                 rx_ring->rx_itr_setting = pf->rx_itr_default;
7692                 vsi->rx_rings[i] = rx_ring;
7693         }
7694
7695         return 0;
7696
7697 err_out:
7698         i40e_vsi_clear_rings(vsi);
7699         return -ENOMEM;
7700 }
7701
7702 /**
7703  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7704  * @pf: board private structure
7705  * @vectors: the number of MSI-X vectors to request
7706  *
7707  * Returns the number of vectors reserved, or error
7708  **/
7709 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7710 {
7711         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7712                                         I40E_MIN_MSIX, vectors);
7713         if (vectors < 0) {
7714                 dev_info(&pf->pdev->dev,
7715                          "MSI-X vector reservation failed: %d\n", vectors);
7716                 vectors = 0;
7717         }
7718
7719         return vectors;
7720 }
7721
7722 /**
7723  * i40e_init_msix - Setup the MSIX capability
7724  * @pf: board private structure
7725  *
7726  * Work with the OS to set up the MSIX vectors needed.
7727  *
7728  * Returns the number of vectors reserved or negative on failure
7729  **/
7730 static int i40e_init_msix(struct i40e_pf *pf)
7731 {
7732         struct i40e_hw *hw = &pf->hw;
7733         int cpus, extra_vectors;
7734         int vectors_left;
7735         int v_budget, i;
7736         int v_actual;
7737         int iwarp_requested = 0;
7738
7739         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7740                 return -ENODEV;
7741
7742         /* The number of vectors we'll request will be comprised of:
7743          *   - Add 1 for "other" cause for Admin Queue events, etc.
7744          *   - The number of LAN queue pairs
7745          *      - Queues being used for RSS.
7746          *              We don't need as many as max_rss_size vectors.
7747          *              use rss_size instead in the calculation since that
7748          *              is governed by number of cpus in the system.
7749          *      - assumes symmetric Tx/Rx pairing
7750          *   - The number of VMDq pairs
7751          *   - The CPU count within the NUMA node if iWARP is enabled
7752          * Once we count this up, try the request.
7753          *
7754          * If we can't get what we want, we'll simplify to nearly nothing
7755          * and try again.  If that still fails, we punt.
7756          */
7757         vectors_left = hw->func_caps.num_msix_vectors;
7758         v_budget = 0;
7759
7760         /* reserve one vector for miscellaneous handler */
7761         if (vectors_left) {
7762                 v_budget++;
7763                 vectors_left--;
7764         }
7765
7766         /* reserve some vectors for the main PF traffic queues. Initially we
7767          * only reserve at most 50% of the available vectors, in the case that
7768          * the number of online CPUs is large. This ensures that we can enable
7769          * extra features as well. Once we've enabled the other features, we
7770          * will use any remaining vectors to reach as close as we can to the
7771          * number of online CPUs.
7772          */
7773         cpus = num_online_cpus();
7774         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
7775         vectors_left -= pf->num_lan_msix;
7776
7777         /* reserve one vector for sideband flow director */
7778         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7779                 if (vectors_left) {
7780                         pf->num_fdsb_msix = 1;
7781                         v_budget++;
7782                         vectors_left--;
7783                 } else {
7784                         pf->num_fdsb_msix = 0;
7785                 }
7786         }
7787
7788         /* can we reserve enough for iWARP? */
7789         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7790                 iwarp_requested = pf->num_iwarp_msix;
7791
7792                 if (!vectors_left)
7793                         pf->num_iwarp_msix = 0;
7794                 else if (vectors_left < pf->num_iwarp_msix)
7795                         pf->num_iwarp_msix = 1;
7796                 v_budget += pf->num_iwarp_msix;
7797                 vectors_left -= pf->num_iwarp_msix;
7798         }
7799
7800         /* any vectors left over go for VMDq support */
7801         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
7802                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
7803                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
7804
7805                 if (!vectors_left) {
7806                         pf->num_vmdq_msix = 0;
7807                         pf->num_vmdq_qps = 0;
7808                 } else {
7809                         /* if we're short on vectors for what's desired, we limit
7810                          * the queues per vmdq.  If this is still more than are
7811                          * available, the user will need to change the number of
7812                          * queues/vectors used by the PF later with the ethtool
7813                          * channels command
7814                          */
7815                         if (vmdq_vecs < vmdq_vecs_wanted)
7816                                 pf->num_vmdq_qps = 1;
7817                         pf->num_vmdq_msix = pf->num_vmdq_qps;
7818
7819                         v_budget += vmdq_vecs;
7820                         vectors_left -= vmdq_vecs;
7821                 }
7822         }
7823
7824         /* On systems with a large number of SMP cores, we previously limited
7825          * the number of vectors for num_lan_msix to be at most 50% of the
7826          * available vectors, to allow for other features. Now, we add back
7827          * the remaining vectors. However, we ensure that the total
7828          * num_lan_msix will not exceed num_online_cpus(). To do this, we
7829          * calculate the number of vectors we can add without going over the
7830          * cap of CPUs. For systems with a small number of CPUs this will be
7831          * zero.
7832          */
7833         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
7834         pf->num_lan_msix += extra_vectors;
7835         vectors_left -= extra_vectors;
7836
7837         WARN(vectors_left < 0,
7838              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
7839
7840         v_budget += pf->num_lan_msix;
7841         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
7842                                    GFP_KERNEL);
7843         if (!pf->msix_entries)
7844                 return -ENOMEM;
7845
7846         for (i = 0; i < v_budget; i++)
7847                 pf->msix_entries[i].entry = i;
7848         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
7849
7850         if (v_actual < I40E_MIN_MSIX) {
7851                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
7852                 kfree(pf->msix_entries);
7853                 pf->msix_entries = NULL;
7854                 pci_disable_msix(pf->pdev);
7855                 return -ENODEV;
7856
7857         } else if (v_actual == I40E_MIN_MSIX) {
7858                 /* Adjust for minimal MSIX use */
7859                 pf->num_vmdq_vsis = 0;
7860                 pf->num_vmdq_qps = 0;
7861                 pf->num_lan_qps = 1;
7862                 pf->num_lan_msix = 1;
7863
7864         } else if (!vectors_left) {
7865                 /* If we have limited resources, we will start with no vectors
7866                  * for the special features and then allocate vectors to some
7867                  * of these features based on the policy and at the end disable
7868                  * the features that did not get any vectors.
7869                  */
7870                 int vec;
7871
7872                 dev_info(&pf->pdev->dev,
7873                          "MSI-X vector limit reached, attempting to redistribute vectors\n");
7874                 /* reserve the misc vector */
7875                 vec = v_actual - 1;
7876
7877                 /* Scale vector usage down */
7878                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
7879                 pf->num_vmdq_vsis = 1;
7880                 pf->num_vmdq_qps = 1;
7881
7882                 /* partition out the remaining vectors */
7883                 switch (vec) {
7884                 case 2:
7885                         pf->num_lan_msix = 1;
7886                         break;
7887                 case 3:
7888                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7889                                 pf->num_lan_msix = 1;
7890                                 pf->num_iwarp_msix = 1;
7891                         } else {
7892                                 pf->num_lan_msix = 2;
7893                         }
7894                         break;
7895                 default:
7896                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7897                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
7898                                                  iwarp_requested);
7899                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
7900                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7901                         } else {
7902                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
7903                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7904                         }
7905                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7906                                 pf->num_fdsb_msix = 1;
7907                                 vec--;
7908                         }
7909                         pf->num_lan_msix = min_t(int,
7910                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
7911                                                               pf->num_lan_msix);
7912                         pf->num_lan_qps = pf->num_lan_msix;
7913                         break;
7914                 }
7915         }
7916
7917         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
7918             (pf->num_fdsb_msix == 0)) {
7919                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
7920                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7921         }
7922         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7923             (pf->num_vmdq_msix == 0)) {
7924                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
7925                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
7926         }
7927
7928         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
7929             (pf->num_iwarp_msix == 0)) {
7930                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
7931                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
7932         }
7933         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
7934                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
7935                    pf->num_lan_msix,
7936                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
7937                    pf->num_fdsb_msix,
7938                    pf->num_iwarp_msix);
7939
7940         return v_actual;
7941 }
7942
7943 /**
7944  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
7945  * @vsi: the VSI being configured
7946  * @v_idx: index of the vector in the vsi struct
7947  * @cpu: cpu to be used on affinity_mask
7948  *
7949  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
7950  **/
7951 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
7952 {
7953         struct i40e_q_vector *q_vector;
7954
7955         /* allocate q_vector */
7956         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
7957         if (!q_vector)
7958                 return -ENOMEM;
7959
7960         q_vector->vsi = vsi;
7961         q_vector->v_idx = v_idx;
7962         cpumask_set_cpu(cpu, &q_vector->affinity_mask);
7963
7964         if (vsi->netdev)
7965                 netif_napi_add(vsi->netdev, &q_vector->napi,
7966                                i40e_napi_poll, NAPI_POLL_WEIGHT);
7967
7968         q_vector->rx.latency_range = I40E_LOW_LATENCY;
7969         q_vector->tx.latency_range = I40E_LOW_LATENCY;
7970
7971         /* tie q_vector and vsi together */
7972         vsi->q_vectors[v_idx] = q_vector;
7973
7974         return 0;
7975 }
7976
7977 /**
7978  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
7979  * @vsi: the VSI being configured
7980  *
7981  * We allocate one q_vector per queue interrupt.  If allocation fails we
7982  * return -ENOMEM.
7983  **/
7984 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
7985 {
7986         struct i40e_pf *pf = vsi->back;
7987         int err, v_idx, num_q_vectors, current_cpu;
7988
7989         /* if not MSIX, give the one vector only to the LAN VSI */
7990         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7991                 num_q_vectors = vsi->num_q_vectors;
7992         else if (vsi == pf->vsi[pf->lan_vsi])
7993                 num_q_vectors = 1;
7994         else
7995                 return -EINVAL;
7996
7997         current_cpu = cpumask_first(cpu_online_mask);
7998
7999         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
8000                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
8001                 if (err)
8002                         goto err_out;
8003                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8004                 if (unlikely(current_cpu >= nr_cpu_ids))
8005                         current_cpu = cpumask_first(cpu_online_mask);
8006         }
8007
8008         return 0;
8009
8010 err_out:
8011         while (v_idx--)
8012                 i40e_free_q_vector(vsi, v_idx);
8013
8014         return err;
8015 }
8016
8017 /**
8018  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8019  * @pf: board private structure to initialize
8020  **/
8021 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8022 {
8023         int vectors = 0;
8024         ssize_t size;
8025
8026         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8027                 vectors = i40e_init_msix(pf);
8028                 if (vectors < 0) {
8029                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
8030                                        I40E_FLAG_IWARP_ENABLED  |
8031                                        I40E_FLAG_RSS_ENABLED    |
8032                                        I40E_FLAG_DCB_CAPABLE    |
8033                                        I40E_FLAG_DCB_ENABLED    |
8034                                        I40E_FLAG_SRIOV_ENABLED  |
8035                                        I40E_FLAG_FD_SB_ENABLED  |
8036                                        I40E_FLAG_FD_ATR_ENABLED |
8037                                        I40E_FLAG_VMDQ_ENABLED);
8038
8039                         /* rework the queue expectations without MSIX */
8040                         i40e_determine_queue_usage(pf);
8041                 }
8042         }
8043
8044         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8045             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8046                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8047                 vectors = pci_enable_msi(pf->pdev);
8048                 if (vectors < 0) {
8049                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8050                                  vectors);
8051                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8052                 }
8053                 vectors = 1;  /* one MSI or Legacy vector */
8054         }
8055
8056         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8057                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8058
8059         /* set up vector assignment tracking */
8060         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8061         pf->irq_pile = kzalloc(size, GFP_KERNEL);
8062         if (!pf->irq_pile) {
8063                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8064                 return -ENOMEM;
8065         }
8066         pf->irq_pile->num_entries = vectors;
8067         pf->irq_pile->search_hint = 0;
8068
8069         /* track first vector for misc interrupts, ignore return */
8070         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8071
8072         return 0;
8073 }
8074
8075 /**
8076  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8077  * @pf: board private structure
8078  *
8079  * This sets up the handler for MSIX 0, which is used to manage the
8080  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
8081  * when in MSI or Legacy interrupt mode.
8082  **/
8083 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8084 {
8085         struct i40e_hw *hw = &pf->hw;
8086         int err = 0;
8087
8088         /* Only request the irq if this is the first time through, and
8089          * not when we're rebuilding after a Reset
8090          */
8091         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
8092                 err = request_irq(pf->msix_entries[0].vector,
8093                                   i40e_intr, 0, pf->int_name, pf);
8094                 if (err) {
8095                         dev_info(&pf->pdev->dev,
8096                                  "request_irq for %s failed: %d\n",
8097                                  pf->int_name, err);
8098                         return -EFAULT;
8099                 }
8100         }
8101
8102         i40e_enable_misc_int_causes(pf);
8103
8104         /* associate no queues to the misc vector */
8105         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8106         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8107
8108         i40e_flush(hw);
8109
8110         i40e_irq_dynamic_enable_icr0(pf, true);
8111
8112         return err;
8113 }
8114
8115 /**
8116  * i40e_config_rss_aq - Prepare for RSS using AQ commands
8117  * @vsi: vsi structure
8118  * @seed: RSS hash seed
8119  **/
8120 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8121                               u8 *lut, u16 lut_size)
8122 {
8123         struct i40e_pf *pf = vsi->back;
8124         struct i40e_hw *hw = &pf->hw;
8125         int ret = 0;
8126
8127         if (seed) {
8128                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8129                         (struct i40e_aqc_get_set_rss_key_data *)seed;
8130                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8131                 if (ret) {
8132                         dev_info(&pf->pdev->dev,
8133                                  "Cannot set RSS key, err %s aq_err %s\n",
8134                                  i40e_stat_str(hw, ret),
8135                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8136                         return ret;
8137                 }
8138         }
8139         if (lut) {
8140                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8141
8142                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8143                 if (ret) {
8144                         dev_info(&pf->pdev->dev,
8145                                  "Cannot set RSS lut, err %s aq_err %s\n",
8146                                  i40e_stat_str(hw, ret),
8147                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8148                         return ret;
8149                 }
8150         }
8151         return ret;
8152 }
8153
8154 /**
8155  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8156  * @vsi: Pointer to vsi structure
8157  * @seed: Buffter to store the hash keys
8158  * @lut: Buffer to store the lookup table entries
8159  * @lut_size: Size of buffer to store the lookup table entries
8160  *
8161  * Return 0 on success, negative on failure
8162  */
8163 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8164                            u8 *lut, u16 lut_size)
8165 {
8166         struct i40e_pf *pf = vsi->back;
8167         struct i40e_hw *hw = &pf->hw;
8168         int ret = 0;
8169
8170         if (seed) {
8171                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8172                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8173                 if (ret) {
8174                         dev_info(&pf->pdev->dev,
8175                                  "Cannot get RSS key, err %s aq_err %s\n",
8176                                  i40e_stat_str(&pf->hw, ret),
8177                                  i40e_aq_str(&pf->hw,
8178                                              pf->hw.aq.asq_last_status));
8179                         return ret;
8180                 }
8181         }
8182
8183         if (lut) {
8184                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8185
8186                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8187                 if (ret) {
8188                         dev_info(&pf->pdev->dev,
8189                                  "Cannot get RSS lut, err %s aq_err %s\n",
8190                                  i40e_stat_str(&pf->hw, ret),
8191                                  i40e_aq_str(&pf->hw,
8192                                              pf->hw.aq.asq_last_status));
8193                         return ret;
8194                 }
8195         }
8196
8197         return ret;
8198 }
8199
8200 /**
8201  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8202  * @vsi: VSI structure
8203  **/
8204 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8205 {
8206         u8 seed[I40E_HKEY_ARRAY_SIZE];
8207         struct i40e_pf *pf = vsi->back;
8208         u8 *lut;
8209         int ret;
8210
8211         if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
8212                 return 0;
8213
8214         if (!vsi->rss_size)
8215                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8216                                       vsi->num_queue_pairs);
8217         if (!vsi->rss_size)
8218                 return -EINVAL;
8219
8220         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8221         if (!lut)
8222                 return -ENOMEM;
8223         /* Use the user configured hash keys and lookup table if there is one,
8224          * otherwise use default
8225          */
8226         if (vsi->rss_lut_user)
8227                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8228         else
8229                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8230         if (vsi->rss_hkey_user)
8231                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8232         else
8233                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8234         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8235         kfree(lut);
8236
8237         return ret;
8238 }
8239
8240 /**
8241  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8242  * @vsi: Pointer to vsi structure
8243  * @seed: RSS hash seed
8244  * @lut: Lookup table
8245  * @lut_size: Lookup table size
8246  *
8247  * Returns 0 on success, negative on failure
8248  **/
8249 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8250                                const u8 *lut, u16 lut_size)
8251 {
8252         struct i40e_pf *pf = vsi->back;
8253         struct i40e_hw *hw = &pf->hw;
8254         u16 vf_id = vsi->vf_id;
8255         u8 i;
8256
8257         /* Fill out hash function seed */
8258         if (seed) {
8259                 u32 *seed_dw = (u32 *)seed;
8260
8261                 if (vsi->type == I40E_VSI_MAIN) {
8262                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8263                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
8264                 } else if (vsi->type == I40E_VSI_SRIOV) {
8265                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8266                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
8267                 } else {
8268                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8269                 }
8270         }
8271
8272         if (lut) {
8273                 u32 *lut_dw = (u32 *)lut;
8274
8275                 if (vsi->type == I40E_VSI_MAIN) {
8276                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
8277                                 return -EINVAL;
8278                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8279                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8280                 } else if (vsi->type == I40E_VSI_SRIOV) {
8281                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8282                                 return -EINVAL;
8283                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8284                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
8285                 } else {
8286                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8287                 }
8288         }
8289         i40e_flush(hw);
8290
8291         return 0;
8292 }
8293
8294 /**
8295  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8296  * @vsi: Pointer to VSI structure
8297  * @seed: Buffer to store the keys
8298  * @lut: Buffer to store the lookup table entries
8299  * @lut_size: Size of buffer to store the lookup table entries
8300  *
8301  * Returns 0 on success, negative on failure
8302  */
8303 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8304                             u8 *lut, u16 lut_size)
8305 {
8306         struct i40e_pf *pf = vsi->back;
8307         struct i40e_hw *hw = &pf->hw;
8308         u16 i;
8309
8310         if (seed) {
8311                 u32 *seed_dw = (u32 *)seed;
8312
8313                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8314                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8315         }
8316         if (lut) {
8317                 u32 *lut_dw = (u32 *)lut;
8318
8319                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8320                         return -EINVAL;
8321                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8322                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8323         }
8324
8325         return 0;
8326 }
8327
8328 /**
8329  * i40e_config_rss - Configure RSS keys and lut
8330  * @vsi: Pointer to VSI structure
8331  * @seed: RSS hash seed
8332  * @lut: Lookup table
8333  * @lut_size: Lookup table size
8334  *
8335  * Returns 0 on success, negative on failure
8336  */
8337 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8338 {
8339         struct i40e_pf *pf = vsi->back;
8340
8341         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8342                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8343         else
8344                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8345 }
8346
8347 /**
8348  * i40e_get_rss - Get RSS keys and lut
8349  * @vsi: Pointer to VSI structure
8350  * @seed: Buffer to store the keys
8351  * @lut: Buffer to store the lookup table entries
8352  * lut_size: Size of buffer to store the lookup table entries
8353  *
8354  * Returns 0 on success, negative on failure
8355  */
8356 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8357 {
8358         struct i40e_pf *pf = vsi->back;
8359
8360         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8361                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8362         else
8363                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8364 }
8365
8366 /**
8367  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8368  * @pf: Pointer to board private structure
8369  * @lut: Lookup table
8370  * @rss_table_size: Lookup table size
8371  * @rss_size: Range of queue number for hashing
8372  */
8373 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8374                        u16 rss_table_size, u16 rss_size)
8375 {
8376         u16 i;
8377
8378         for (i = 0; i < rss_table_size; i++)
8379                 lut[i] = i % rss_size;
8380 }
8381
8382 /**
8383  * i40e_pf_config_rss - Prepare for RSS if used
8384  * @pf: board private structure
8385  **/
8386 static int i40e_pf_config_rss(struct i40e_pf *pf)
8387 {
8388         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8389         u8 seed[I40E_HKEY_ARRAY_SIZE];
8390         u8 *lut;
8391         struct i40e_hw *hw = &pf->hw;
8392         u32 reg_val;
8393         u64 hena;
8394         int ret;
8395
8396         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8397         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8398                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8399         hena |= i40e_pf_get_default_rss_hena(pf);
8400
8401         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8402         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8403
8404         /* Determine the RSS table size based on the hardware capabilities */
8405         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8406         reg_val = (pf->rss_table_size == 512) ?
8407                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8408                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8409         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8410
8411         /* Determine the RSS size of the VSI */
8412         if (!vsi->rss_size) {
8413                 u16 qcount;
8414
8415                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8416                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8417         }
8418         if (!vsi->rss_size)
8419                 return -EINVAL;
8420
8421         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8422         if (!lut)
8423                 return -ENOMEM;
8424
8425         /* Use user configured lut if there is one, otherwise use default */
8426         if (vsi->rss_lut_user)
8427                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8428         else
8429                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8430
8431         /* Use user configured hash key if there is one, otherwise
8432          * use default.
8433          */
8434         if (vsi->rss_hkey_user)
8435                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8436         else
8437                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8438         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8439         kfree(lut);
8440
8441         return ret;
8442 }
8443
8444 /**
8445  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8446  * @pf: board private structure
8447  * @queue_count: the requested queue count for rss.
8448  *
8449  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8450  * count which may be different from the requested queue count.
8451  * Note: expects to be called while under rtnl_lock()
8452  **/
8453 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8454 {
8455         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8456         int new_rss_size;
8457
8458         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8459                 return 0;
8460
8461         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8462
8463         if (queue_count != vsi->num_queue_pairs) {
8464                 u16 qcount;
8465
8466                 vsi->req_queue_pairs = queue_count;
8467                 i40e_prep_for_reset(pf, true);
8468
8469                 pf->alloc_rss_size = new_rss_size;
8470
8471                 i40e_reset_and_rebuild(pf, true, true);
8472
8473                 /* Discard the user configured hash keys and lut, if less
8474                  * queues are enabled.
8475                  */
8476                 if (queue_count < vsi->rss_size) {
8477                         i40e_clear_rss_config_user(vsi);
8478                         dev_dbg(&pf->pdev->dev,
8479                                 "discard user configured hash keys and lut\n");
8480                 }
8481
8482                 /* Reset vsi->rss_size, as number of enabled queues changed */
8483                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
8484                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
8485
8486                 i40e_pf_config_rss(pf);
8487         }
8488         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
8489                  vsi->req_queue_pairs, pf->rss_size_max);
8490         return pf->alloc_rss_size;
8491 }
8492
8493 /**
8494  * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition
8495  * @pf: board private structure
8496  **/
8497 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf)
8498 {
8499         i40e_status status;
8500         bool min_valid, max_valid;
8501         u32 max_bw, min_bw;
8502
8503         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8504                                            &min_valid, &max_valid);
8505
8506         if (!status) {
8507                 if (min_valid)
8508                         pf->npar_min_bw = min_bw;
8509                 if (max_valid)
8510                         pf->npar_max_bw = max_bw;
8511         }
8512
8513         return status;
8514 }
8515
8516 /**
8517  * i40e_set_npar_bw_setting - Set BW settings for this PF partition
8518  * @pf: board private structure
8519  **/
8520 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf)
8521 {
8522         struct i40e_aqc_configure_partition_bw_data bw_data;
8523         i40e_status status;
8524
8525         /* Set the valid bit for this PF */
8526         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8527         bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK;
8528         bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK;
8529
8530         /* Set the new bandwidths */
8531         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8532
8533         return status;
8534 }
8535
8536 /**
8537  * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition
8538  * @pf: board private structure
8539  **/
8540 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf)
8541 {
8542         /* Commit temporary BW setting to permanent NVM image */
8543         enum i40e_admin_queue_err last_aq_status;
8544         i40e_status ret;
8545         u16 nvm_word;
8546
8547         if (pf->hw.partition_id != 1) {
8548                 dev_info(&pf->pdev->dev,
8549                          "Commit BW only works on partition 1! This is partition %d",
8550                          pf->hw.partition_id);
8551                 ret = I40E_NOT_SUPPORTED;
8552                 goto bw_commit_out;
8553         }
8554
8555         /* Acquire NVM for read access */
8556         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8557         last_aq_status = pf->hw.aq.asq_last_status;
8558         if (ret) {
8559                 dev_info(&pf->pdev->dev,
8560                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8561                          i40e_stat_str(&pf->hw, ret),
8562                          i40e_aq_str(&pf->hw, last_aq_status));
8563                 goto bw_commit_out;
8564         }
8565
8566         /* Read word 0x10 of NVM - SW compatibility word 1 */
8567         ret = i40e_aq_read_nvm(&pf->hw,
8568                                I40E_SR_NVM_CONTROL_WORD,
8569                                0x10, sizeof(nvm_word), &nvm_word,
8570                                false, NULL);
8571         /* Save off last admin queue command status before releasing
8572          * the NVM
8573          */
8574         last_aq_status = pf->hw.aq.asq_last_status;
8575         i40e_release_nvm(&pf->hw);
8576         if (ret) {
8577                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8578                          i40e_stat_str(&pf->hw, ret),
8579                          i40e_aq_str(&pf->hw, last_aq_status));
8580                 goto bw_commit_out;
8581         }
8582
8583         /* Wait a bit for NVM release to complete */
8584         msleep(50);
8585
8586         /* Acquire NVM for write access */
8587         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8588         last_aq_status = pf->hw.aq.asq_last_status;
8589         if (ret) {
8590                 dev_info(&pf->pdev->dev,
8591                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8592                          i40e_stat_str(&pf->hw, ret),
8593                          i40e_aq_str(&pf->hw, last_aq_status));
8594                 goto bw_commit_out;
8595         }
8596         /* Write it back out unchanged to initiate update NVM,
8597          * which will force a write of the shadow (alt) RAM to
8598          * the NVM - thus storing the bandwidth values permanently.
8599          */
8600         ret = i40e_aq_update_nvm(&pf->hw,
8601                                  I40E_SR_NVM_CONTROL_WORD,
8602                                  0x10, sizeof(nvm_word),
8603                                  &nvm_word, true, NULL);
8604         /* Save off last admin queue command status before releasing
8605          * the NVM
8606          */
8607         last_aq_status = pf->hw.aq.asq_last_status;
8608         i40e_release_nvm(&pf->hw);
8609         if (ret)
8610                 dev_info(&pf->pdev->dev,
8611                          "BW settings NOT SAVED, err %s aq_err %s\n",
8612                          i40e_stat_str(&pf->hw, ret),
8613                          i40e_aq_str(&pf->hw, last_aq_status));
8614 bw_commit_out:
8615
8616         return ret;
8617 }
8618
8619 /**
8620  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8621  * @pf: board private structure to initialize
8622  *
8623  * i40e_sw_init initializes the Adapter private data structure.
8624  * Fields are initialized based on PCI device information and
8625  * OS network device settings (MTU size).
8626  **/
8627 static int i40e_sw_init(struct i40e_pf *pf)
8628 {
8629         int err = 0;
8630         int size;
8631
8632         /* Set default capability flags */
8633         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8634                     I40E_FLAG_MSI_ENABLED     |
8635                     I40E_FLAG_MSIX_ENABLED;
8636
8637         /* Set default ITR */
8638         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8639         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8640
8641         /* Depending on PF configurations, it is possible that the RSS
8642          * maximum might end up larger than the available queues
8643          */
8644         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8645         pf->alloc_rss_size = 1;
8646         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8647         pf->rss_size_max = min_t(int, pf->rss_size_max,
8648                                  pf->hw.func_caps.num_tx_qp);
8649         if (pf->hw.func_caps.rss) {
8650                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8651                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8652                                            num_online_cpus());
8653         }
8654
8655         /* MFP mode enabled */
8656         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8657                 pf->flags |= I40E_FLAG_MFP_ENABLED;
8658                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8659                 if (i40e_get_npar_bw_setting(pf))
8660                         dev_warn(&pf->pdev->dev,
8661                                  "Could not get NPAR bw settings\n");
8662                 else
8663                         dev_info(&pf->pdev->dev,
8664                                  "Min BW = %8.8x, Max BW = %8.8x\n",
8665                                  pf->npar_min_bw, pf->npar_max_bw);
8666         }
8667
8668         /* FW/NVM is not yet fixed in this regard */
8669         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8670             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8671                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8672                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8673                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8674                     pf->hw.num_partitions > 1)
8675                         dev_info(&pf->pdev->dev,
8676                                  "Flow Director Sideband mode Disabled in MFP mode\n");
8677                 else
8678                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8679                 pf->fdir_pf_filter_count =
8680                                  pf->hw.func_caps.fd_filters_guaranteed;
8681                 pf->hw.fdir_shared_filter_count =
8682                                  pf->hw.func_caps.fd_filters_best_effort;
8683         }
8684
8685         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8686             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
8687             (pf->hw.aq.fw_maj_ver < 4))) {
8688                 pf->flags |= I40E_FLAG_RESTART_AUTONEG;
8689                 /* No DCB support  for FW < v4.33 */
8690                 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT;
8691         }
8692
8693         /* Disable FW LLDP if FW < v4.3 */
8694         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8695             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
8696             (pf->hw.aq.fw_maj_ver < 4)))
8697                 pf->flags |= I40E_FLAG_STOP_FW_LLDP;
8698
8699         /* Use the FW Set LLDP MIB API if FW > v4.40 */
8700         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
8701             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
8702             (pf->hw.aq.fw_maj_ver >= 5)))
8703                 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB;
8704
8705         if (pf->hw.func_caps.vmdq) {
8706                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
8707                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
8708                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
8709         }
8710
8711         if (pf->hw.func_caps.iwarp) {
8712                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
8713                 /* IWARP needs one extra vector for CQP just like MISC.*/
8714                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
8715         }
8716
8717 #ifdef CONFIG_PCI_IOV
8718         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
8719                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
8720                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
8721                 pf->num_req_vfs = min_t(int,
8722                                         pf->hw.func_caps.num_vfs,
8723                                         I40E_MAX_VF_COUNT);
8724         }
8725 #endif /* CONFIG_PCI_IOV */
8726         if (pf->hw.mac.type == I40E_MAC_X722) {
8727                 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE
8728                              | I40E_FLAG_128_QP_RSS_CAPABLE
8729                              | I40E_FLAG_HW_ATR_EVICT_CAPABLE
8730                              | I40E_FLAG_OUTER_UDP_CSUM_CAPABLE
8731                              | I40E_FLAG_WB_ON_ITR_CAPABLE
8732                              | I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE
8733                              | I40E_FLAG_NO_PCI_LINK_CHECK
8734                              | I40E_FLAG_USE_SET_LLDP_MIB
8735                              | I40E_FLAG_GENEVE_OFFLOAD_CAPABLE
8736                              | I40E_FLAG_PTP_L4_CAPABLE
8737                              | I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE;
8738         } else if ((pf->hw.aq.api_maj_ver > 1) ||
8739                    ((pf->hw.aq.api_maj_ver == 1) &&
8740                     (pf->hw.aq.api_min_ver > 4))) {
8741                 /* Supported in FW API version higher than 1.4 */
8742                 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
8743                 pf->hw_disabled_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8744         } else {
8745                 pf->hw_disabled_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8746         }
8747
8748         pf->eeprom_version = 0xDEAD;
8749         pf->lan_veb = I40E_NO_VEB;
8750         pf->lan_vsi = I40E_NO_VSI;
8751
8752         /* By default FW has this off for performance reasons */
8753         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
8754
8755         /* set up queue assignment tracking */
8756         size = sizeof(struct i40e_lump_tracking)
8757                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
8758         pf->qp_pile = kzalloc(size, GFP_KERNEL);
8759         if (!pf->qp_pile) {
8760                 err = -ENOMEM;
8761                 goto sw_init_done;
8762         }
8763         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
8764         pf->qp_pile->search_hint = 0;
8765
8766         pf->tx_timeout_recovery_level = 1;
8767
8768         mutex_init(&pf->switch_mutex);
8769
8770         /* If NPAR is enabled nudge the Tx scheduler */
8771         if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf)))
8772                 i40e_set_npar_bw_setting(pf);
8773
8774 sw_init_done:
8775         return err;
8776 }
8777
8778 /**
8779  * i40e_set_ntuple - set the ntuple feature flag and take action
8780  * @pf: board private structure to initialize
8781  * @features: the feature set that the stack is suggesting
8782  *
8783  * returns a bool to indicate if reset needs to happen
8784  **/
8785 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
8786 {
8787         bool need_reset = false;
8788
8789         /* Check if Flow Director n-tuple support was enabled or disabled.  If
8790          * the state changed, we need to reset.
8791          */
8792         if (features & NETIF_F_NTUPLE) {
8793                 /* Enable filters and mark for reset */
8794                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
8795                         need_reset = true;
8796                 /* enable FD_SB only if there is MSI-X vector */
8797                 if (pf->num_fdsb_msix > 0)
8798                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8799         } else {
8800                 /* turn off filters, mark for reset and clear SW filter list */
8801                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8802                         need_reset = true;
8803                         i40e_fdir_filter_exit(pf);
8804                 }
8805                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8806                 pf->hw_disabled_flags &= ~I40E_FLAG_FD_SB_ENABLED;
8807                 /* reset fd counters */
8808                 pf->fd_add_err = 0;
8809                 pf->fd_atr_cnt = 0;
8810                 /* if ATR was auto disabled it can be re-enabled. */
8811                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8812                     (pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED)) {
8813                         pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
8814                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
8815                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
8816                 }
8817         }
8818         return need_reset;
8819 }
8820
8821 /**
8822  * i40e_clear_rss_lut - clear the rx hash lookup table
8823  * @vsi: the VSI being configured
8824  **/
8825 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
8826 {
8827         struct i40e_pf *pf = vsi->back;
8828         struct i40e_hw *hw = &pf->hw;
8829         u16 vf_id = vsi->vf_id;
8830         u8 i;
8831
8832         if (vsi->type == I40E_VSI_MAIN) {
8833                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8834                         wr32(hw, I40E_PFQF_HLUT(i), 0);
8835         } else if (vsi->type == I40E_VSI_SRIOV) {
8836                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8837                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
8838         } else {
8839                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8840         }
8841 }
8842
8843 /**
8844  * i40e_set_features - set the netdev feature flags
8845  * @netdev: ptr to the netdev being adjusted
8846  * @features: the feature set that the stack is suggesting
8847  * Note: expects to be called while under rtnl_lock()
8848  **/
8849 static int i40e_set_features(struct net_device *netdev,
8850                              netdev_features_t features)
8851 {
8852         struct i40e_netdev_priv *np = netdev_priv(netdev);
8853         struct i40e_vsi *vsi = np->vsi;
8854         struct i40e_pf *pf = vsi->back;
8855         bool need_reset;
8856
8857         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
8858                 i40e_pf_config_rss(pf);
8859         else if (!(features & NETIF_F_RXHASH) &&
8860                  netdev->features & NETIF_F_RXHASH)
8861                 i40e_clear_rss_lut(vsi);
8862
8863         if (features & NETIF_F_HW_VLAN_CTAG_RX)
8864                 i40e_vlan_stripping_enable(vsi);
8865         else
8866                 i40e_vlan_stripping_disable(vsi);
8867
8868         need_reset = i40e_set_ntuple(pf, features);
8869
8870         if (need_reset)
8871                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true);
8872
8873         return 0;
8874 }
8875
8876 /**
8877  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
8878  * @pf: board private structure
8879  * @port: The UDP port to look up
8880  *
8881  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
8882  **/
8883 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
8884 {
8885         u8 i;
8886
8887         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
8888                 if (pf->udp_ports[i].index == port)
8889                         return i;
8890         }
8891
8892         return i;
8893 }
8894
8895 /**
8896  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
8897  * @netdev: This physical port's netdev
8898  * @ti: Tunnel endpoint information
8899  **/
8900 static void i40e_udp_tunnel_add(struct net_device *netdev,
8901                                 struct udp_tunnel_info *ti)
8902 {
8903         struct i40e_netdev_priv *np = netdev_priv(netdev);
8904         struct i40e_vsi *vsi = np->vsi;
8905         struct i40e_pf *pf = vsi->back;
8906         u16 port = ntohs(ti->port);
8907         u8 next_idx;
8908         u8 idx;
8909
8910         idx = i40e_get_udp_port_idx(pf, port);
8911
8912         /* Check if port already exists */
8913         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8914                 netdev_info(netdev, "port %d already offloaded\n", port);
8915                 return;
8916         }
8917
8918         /* Now check if there is space to add the new port */
8919         next_idx = i40e_get_udp_port_idx(pf, 0);
8920
8921         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8922                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
8923                             port);
8924                 return;
8925         }
8926
8927         switch (ti->type) {
8928         case UDP_TUNNEL_TYPE_VXLAN:
8929                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
8930                 break;
8931         case UDP_TUNNEL_TYPE_GENEVE:
8932                 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
8933                         return;
8934                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
8935                 break;
8936         default:
8937                 return;
8938         }
8939
8940         /* New port: add it and mark its index in the bitmap */
8941         pf->udp_ports[next_idx].index = port;
8942         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
8943         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8944 }
8945
8946 /**
8947  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
8948  * @netdev: This physical port's netdev
8949  * @ti: Tunnel endpoint information
8950  **/
8951 static void i40e_udp_tunnel_del(struct net_device *netdev,
8952                                 struct udp_tunnel_info *ti)
8953 {
8954         struct i40e_netdev_priv *np = netdev_priv(netdev);
8955         struct i40e_vsi *vsi = np->vsi;
8956         struct i40e_pf *pf = vsi->back;
8957         u16 port = ntohs(ti->port);
8958         u8 idx;
8959
8960         idx = i40e_get_udp_port_idx(pf, port);
8961
8962         /* Check if port already exists */
8963         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
8964                 goto not_found;
8965
8966         switch (ti->type) {
8967         case UDP_TUNNEL_TYPE_VXLAN:
8968                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
8969                         goto not_found;
8970                 break;
8971         case UDP_TUNNEL_TYPE_GENEVE:
8972                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
8973                         goto not_found;
8974                 break;
8975         default:
8976                 goto not_found;
8977         }
8978
8979         /* if port exists, set it to 0 (mark for deletion)
8980          * and make it pending
8981          */
8982         pf->udp_ports[idx].index = 0;
8983         pf->pending_udp_bitmap |= BIT_ULL(idx);
8984         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8985
8986         return;
8987 not_found:
8988         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
8989                     port);
8990 }
8991
8992 static int i40e_get_phys_port_id(struct net_device *netdev,
8993                                  struct netdev_phys_item_id *ppid)
8994 {
8995         struct i40e_netdev_priv *np = netdev_priv(netdev);
8996         struct i40e_pf *pf = np->vsi->back;
8997         struct i40e_hw *hw = &pf->hw;
8998
8999         if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
9000                 return -EOPNOTSUPP;
9001
9002         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9003         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9004
9005         return 0;
9006 }
9007
9008 /**
9009  * i40e_ndo_fdb_add - add an entry to the hardware database
9010  * @ndm: the input from the stack
9011  * @tb: pointer to array of nladdr (unused)
9012  * @dev: the net device pointer
9013  * @addr: the MAC address entry being added
9014  * @flags: instructions from stack about fdb operation
9015  */
9016 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9017                             struct net_device *dev,
9018                             const unsigned char *addr, u16 vid,
9019                             u16 flags)
9020 {
9021         struct i40e_netdev_priv *np = netdev_priv(dev);
9022         struct i40e_pf *pf = np->vsi->back;
9023         int err = 0;
9024
9025         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9026                 return -EOPNOTSUPP;
9027
9028         if (vid) {
9029                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9030                 return -EINVAL;
9031         }
9032
9033         /* Hardware does not support aging addresses so if a
9034          * ndm_state is given only allow permanent addresses
9035          */
9036         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9037                 netdev_info(dev, "FDB only supports static addresses\n");
9038                 return -EINVAL;
9039         }
9040
9041         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9042                 err = dev_uc_add_excl(dev, addr);
9043         else if (is_multicast_ether_addr(addr))
9044                 err = dev_mc_add_excl(dev, addr);
9045         else
9046                 err = -EINVAL;
9047
9048         /* Only return duplicate errors if NLM_F_EXCL is set */
9049         if (err == -EEXIST && !(flags & NLM_F_EXCL))
9050                 err = 0;
9051
9052         return err;
9053 }
9054
9055 /**
9056  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9057  * @dev: the netdev being configured
9058  * @nlh: RTNL message
9059  *
9060  * Inserts a new hardware bridge if not already created and
9061  * enables the bridging mode requested (VEB or VEPA). If the
9062  * hardware bridge has already been inserted and the request
9063  * is to change the mode then that requires a PF reset to
9064  * allow rebuild of the components with required hardware
9065  * bridge mode enabled.
9066  *
9067  * Note: expects to be called while under rtnl_lock()
9068  **/
9069 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9070                                    struct nlmsghdr *nlh,
9071                                    u16 flags)
9072 {
9073         struct i40e_netdev_priv *np = netdev_priv(dev);
9074         struct i40e_vsi *vsi = np->vsi;
9075         struct i40e_pf *pf = vsi->back;
9076         struct i40e_veb *veb = NULL;
9077         struct nlattr *attr, *br_spec;
9078         int i, rem;
9079
9080         /* Only for PF VSI for now */
9081         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9082                 return -EOPNOTSUPP;
9083
9084         /* Find the HW bridge for PF VSI */
9085         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9086                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9087                         veb = pf->veb[i];
9088         }
9089
9090         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9091
9092         nla_for_each_nested(attr, br_spec, rem) {
9093                 __u16 mode;
9094
9095                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9096                         continue;
9097
9098                 mode = nla_get_u16(attr);
9099                 if ((mode != BRIDGE_MODE_VEPA) &&
9100                     (mode != BRIDGE_MODE_VEB))
9101                         return -EINVAL;
9102
9103                 /* Insert a new HW bridge */
9104                 if (!veb) {
9105                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9106                                              vsi->tc_config.enabled_tc);
9107                         if (veb) {
9108                                 veb->bridge_mode = mode;
9109                                 i40e_config_bridge_mode(veb);
9110                         } else {
9111                                 /* No Bridge HW offload available */
9112                                 return -ENOENT;
9113                         }
9114                         break;
9115                 } else if (mode != veb->bridge_mode) {
9116                         /* Existing HW bridge but different mode needs reset */
9117                         veb->bridge_mode = mode;
9118                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9119                         if (mode == BRIDGE_MODE_VEB)
9120                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9121                         else
9122                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9123                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED),
9124                                       true);
9125                         break;
9126                 }
9127         }
9128
9129         return 0;
9130 }
9131
9132 /**
9133  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9134  * @skb: skb buff
9135  * @pid: process id
9136  * @seq: RTNL message seq #
9137  * @dev: the netdev being configured
9138  * @filter_mask: unused
9139  * @nlflags: netlink flags passed in
9140  *
9141  * Return the mode in which the hardware bridge is operating in
9142  * i.e VEB or VEPA.
9143  **/
9144 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9145                                    struct net_device *dev,
9146                                    u32 __always_unused filter_mask,
9147                                    int nlflags)
9148 {
9149         struct i40e_netdev_priv *np = netdev_priv(dev);
9150         struct i40e_vsi *vsi = np->vsi;
9151         struct i40e_pf *pf = vsi->back;
9152         struct i40e_veb *veb = NULL;
9153         int i;
9154
9155         /* Only for PF VSI for now */
9156         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9157                 return -EOPNOTSUPP;
9158
9159         /* Find the HW bridge for the PF VSI */
9160         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9161                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9162                         veb = pf->veb[i];
9163         }
9164
9165         if (!veb)
9166                 return 0;
9167
9168         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9169                                        0, 0, nlflags, filter_mask, NULL);
9170 }
9171
9172 /**
9173  * i40e_features_check - Validate encapsulated packet conforms to limits
9174  * @skb: skb buff
9175  * @dev: This physical port's netdev
9176  * @features: Offload features that the stack believes apply
9177  **/
9178 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9179                                              struct net_device *dev,
9180                                              netdev_features_t features)
9181 {
9182         size_t len;
9183
9184         /* No point in doing any of this if neither checksum nor GSO are
9185          * being requested for this frame.  We can rule out both by just
9186          * checking for CHECKSUM_PARTIAL
9187          */
9188         if (skb->ip_summed != CHECKSUM_PARTIAL)
9189                 return features;
9190
9191         /* We cannot support GSO if the MSS is going to be less than
9192          * 64 bytes.  If it is then we need to drop support for GSO.
9193          */
9194         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9195                 features &= ~NETIF_F_GSO_MASK;
9196
9197         /* MACLEN can support at most 63 words */
9198         len = skb_network_header(skb) - skb->data;
9199         if (len & ~(63 * 2))
9200                 goto out_err;
9201
9202         /* IPLEN and EIPLEN can support at most 127 dwords */
9203         len = skb_transport_header(skb) - skb_network_header(skb);
9204         if (len & ~(127 * 4))
9205                 goto out_err;
9206
9207         if (skb->encapsulation) {
9208                 /* L4TUNLEN can support 127 words */
9209                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9210                 if (len & ~(127 * 2))
9211                         goto out_err;
9212
9213                 /* IPLEN can support at most 127 dwords */
9214                 len = skb_inner_transport_header(skb) -
9215                       skb_inner_network_header(skb);
9216                 if (len & ~(127 * 4))
9217                         goto out_err;
9218         }
9219
9220         /* No need to validate L4LEN as TCP is the only protocol with a
9221          * a flexible value and we support all possible values supported
9222          * by TCP, which is at most 15 dwords
9223          */
9224
9225         return features;
9226 out_err:
9227         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9228 }
9229
9230 static const struct net_device_ops i40e_netdev_ops = {
9231         .ndo_open               = i40e_open,
9232         .ndo_stop               = i40e_close,
9233         .ndo_start_xmit         = i40e_lan_xmit_frame,
9234         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9235         .ndo_set_rx_mode        = i40e_set_rx_mode,
9236         .ndo_validate_addr      = eth_validate_addr,
9237         .ndo_set_mac_address    = i40e_set_mac,
9238         .ndo_change_mtu         = i40e_change_mtu,
9239         .ndo_do_ioctl           = i40e_ioctl,
9240         .ndo_tx_timeout         = i40e_tx_timeout,
9241         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9242         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9243 #ifdef CONFIG_NET_POLL_CONTROLLER
9244         .ndo_poll_controller    = i40e_netpoll,
9245 #endif
9246         .ndo_setup_tc           = __i40e_setup_tc,
9247         .ndo_set_features       = i40e_set_features,
9248         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9249         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9250         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9251         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9252         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9253         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9254         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9255         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
9256         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
9257         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9258         .ndo_fdb_add            = i40e_ndo_fdb_add,
9259         .ndo_features_check     = i40e_features_check,
9260         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9261         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9262 };
9263
9264 /**
9265  * i40e_config_netdev - Setup the netdev flags
9266  * @vsi: the VSI being configured
9267  *
9268  * Returns 0 on success, negative value on failure
9269  **/
9270 static int i40e_config_netdev(struct i40e_vsi *vsi)
9271 {
9272         struct i40e_pf *pf = vsi->back;
9273         struct i40e_hw *hw = &pf->hw;
9274         struct i40e_netdev_priv *np;
9275         struct net_device *netdev;
9276         u8 broadcast[ETH_ALEN];
9277         u8 mac_addr[ETH_ALEN];
9278         int etherdev_size;
9279         netdev_features_t hw_enc_features;
9280         netdev_features_t hw_features;
9281
9282         etherdev_size = sizeof(struct i40e_netdev_priv);
9283         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9284         if (!netdev)
9285                 return -ENOMEM;
9286
9287         vsi->netdev = netdev;
9288         np = netdev_priv(netdev);
9289         np->vsi = vsi;
9290
9291         hw_enc_features = NETIF_F_SG                    |
9292                           NETIF_F_IP_CSUM               |
9293                           NETIF_F_IPV6_CSUM             |
9294                           NETIF_F_HIGHDMA               |
9295                           NETIF_F_SOFT_FEATURES         |
9296                           NETIF_F_TSO                   |
9297                           NETIF_F_TSO_ECN               |
9298                           NETIF_F_TSO6                  |
9299                           NETIF_F_GSO_GRE               |
9300                           NETIF_F_GSO_GRE_CSUM          |
9301                           NETIF_F_GSO_PARTIAL           |
9302                           NETIF_F_GSO_UDP_TUNNEL        |
9303                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
9304                           NETIF_F_SCTP_CRC              |
9305                           NETIF_F_RXHASH                |
9306                           NETIF_F_RXCSUM                |
9307                           0;
9308
9309         if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE))
9310                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9311
9312         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9313
9314         netdev->hw_enc_features |= hw_enc_features;
9315
9316         /* record features VLANs can make use of */
9317         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
9318
9319         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9320                 netdev->hw_features |= NETIF_F_NTUPLE;
9321         hw_features = hw_enc_features           |
9322                       NETIF_F_HW_VLAN_CTAG_TX   |
9323                       NETIF_F_HW_VLAN_CTAG_RX;
9324
9325         netdev->hw_features |= hw_features;
9326
9327         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9328         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9329
9330         if (vsi->type == I40E_VSI_MAIN) {
9331                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9332                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9333                 /* The following steps are necessary for two reasons. First,
9334                  * some older NVM configurations load a default MAC-VLAN
9335                  * filter that will accept any tagged packet, and we want to
9336                  * replace this with a normal filter. Additionally, it is
9337                  * possible our MAC address was provided by the platform using
9338                  * Open Firmware or similar.
9339                  *
9340                  * Thus, we need to remove the default filter and install one
9341                  * specific to the MAC address.
9342                  */
9343                 i40e_rm_default_mac_filter(vsi, mac_addr);
9344                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9345                 i40e_add_mac_filter(vsi, mac_addr);
9346                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9347         } else {
9348                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
9349                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
9350                          pf->vsi[pf->lan_vsi]->netdev->name);
9351                 random_ether_addr(mac_addr);
9352
9353                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9354                 i40e_add_mac_filter(vsi, mac_addr);
9355                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9356         }
9357
9358         /* Add the broadcast filter so that we initially will receive
9359          * broadcast packets. Note that when a new VLAN is first added the
9360          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9361          * specific filters as part of transitioning into "vlan" operation.
9362          * When more VLANs are added, the driver will copy each existing MAC
9363          * filter and add it for the new VLAN.
9364          *
9365          * Broadcast filters are handled specially by
9366          * i40e_sync_filters_subtask, as the driver must to set the broadcast
9367          * promiscuous bit instead of adding this directly as a MAC/VLAN
9368          * filter. The subtask will update the correct broadcast promiscuous
9369          * bits as VLANs become active or inactive.
9370          */
9371         eth_broadcast_addr(broadcast);
9372         spin_lock_bh(&vsi->mac_filter_hash_lock);
9373         i40e_add_mac_filter(vsi, broadcast);
9374         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9375
9376         ether_addr_copy(netdev->dev_addr, mac_addr);
9377         ether_addr_copy(netdev->perm_addr, mac_addr);
9378
9379         netdev->priv_flags |= IFF_UNICAST_FLT;
9380         netdev->priv_flags |= IFF_SUPP_NOFCS;
9381         /* Setup netdev TC information */
9382         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9383
9384         netdev->netdev_ops = &i40e_netdev_ops;
9385         netdev->watchdog_timeo = 5 * HZ;
9386         i40e_set_ethtool_ops(netdev);
9387
9388         /* MTU range: 68 - 9706 */
9389         netdev->min_mtu = ETH_MIN_MTU;
9390         netdev->max_mtu = I40E_MAX_RXBUFFER -
9391                           (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
9392
9393         return 0;
9394 }
9395
9396 /**
9397  * i40e_vsi_delete - Delete a VSI from the switch
9398  * @vsi: the VSI being removed
9399  *
9400  * Returns 0 on success, negative value on failure
9401  **/
9402 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9403 {
9404         /* remove default VSI is not allowed */
9405         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9406                 return;
9407
9408         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9409 }
9410
9411 /**
9412  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9413  * @vsi: the VSI being queried
9414  *
9415  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9416  **/
9417 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9418 {
9419         struct i40e_veb *veb;
9420         struct i40e_pf *pf = vsi->back;
9421
9422         /* Uplink is not a bridge so default to VEB */
9423         if (vsi->veb_idx == I40E_NO_VEB)
9424                 return 1;
9425
9426         veb = pf->veb[vsi->veb_idx];
9427         if (!veb) {
9428                 dev_info(&pf->pdev->dev,
9429                          "There is no veb associated with the bridge\n");
9430                 return -ENOENT;
9431         }
9432
9433         /* Uplink is a bridge in VEPA mode */
9434         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9435                 return 0;
9436         } else {
9437                 /* Uplink is a bridge in VEB mode */
9438                 return 1;
9439         }
9440
9441         /* VEPA is now default bridge, so return 0 */
9442         return 0;
9443 }
9444
9445 /**
9446  * i40e_add_vsi - Add a VSI to the switch
9447  * @vsi: the VSI being configured
9448  *
9449  * This initializes a VSI context depending on the VSI type to be added and
9450  * passes it down to the add_vsi aq command.
9451  **/
9452 static int i40e_add_vsi(struct i40e_vsi *vsi)
9453 {
9454         int ret = -ENODEV;
9455         struct i40e_pf *pf = vsi->back;
9456         struct i40e_hw *hw = &pf->hw;
9457         struct i40e_vsi_context ctxt;
9458         struct i40e_mac_filter *f;
9459         struct hlist_node *h;
9460         int bkt;
9461
9462         u8 enabled_tc = 0x1; /* TC0 enabled */
9463         int f_count = 0;
9464
9465         memset(&ctxt, 0, sizeof(ctxt));
9466         switch (vsi->type) {
9467         case I40E_VSI_MAIN:
9468                 /* The PF's main VSI is already setup as part of the
9469                  * device initialization, so we'll not bother with
9470                  * the add_vsi call, but we will retrieve the current
9471                  * VSI context.
9472                  */
9473                 ctxt.seid = pf->main_vsi_seid;
9474                 ctxt.pf_num = pf->hw.pf_id;
9475                 ctxt.vf_num = 0;
9476                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9477                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9478                 if (ret) {
9479                         dev_info(&pf->pdev->dev,
9480                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9481                                  i40e_stat_str(&pf->hw, ret),
9482                                  i40e_aq_str(&pf->hw,
9483                                              pf->hw.aq.asq_last_status));
9484                         return -ENOENT;
9485                 }
9486                 vsi->info = ctxt.info;
9487                 vsi->info.valid_sections = 0;
9488
9489                 vsi->seid = ctxt.seid;
9490                 vsi->id = ctxt.vsi_number;
9491
9492                 enabled_tc = i40e_pf_get_tc_map(pf);
9493
9494                 /* MFP mode setup queue map and update VSI */
9495                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9496                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9497                         memset(&ctxt, 0, sizeof(ctxt));
9498                         ctxt.seid = pf->main_vsi_seid;
9499                         ctxt.pf_num = pf->hw.pf_id;
9500                         ctxt.vf_num = 0;
9501                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9502                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9503                         if (ret) {
9504                                 dev_info(&pf->pdev->dev,
9505                                          "update vsi failed, err %s aq_err %s\n",
9506                                          i40e_stat_str(&pf->hw, ret),
9507                                          i40e_aq_str(&pf->hw,
9508                                                     pf->hw.aq.asq_last_status));
9509                                 ret = -ENOENT;
9510                                 goto err;
9511                         }
9512                         /* update the local VSI info queue map */
9513                         i40e_vsi_update_queue_map(vsi, &ctxt);
9514                         vsi->info.valid_sections = 0;
9515                 } else {
9516                         /* Default/Main VSI is only enabled for TC0
9517                          * reconfigure it to enable all TCs that are
9518                          * available on the port in SFP mode.
9519                          * For MFP case the iSCSI PF would use this
9520                          * flow to enable LAN+iSCSI TC.
9521                          */
9522                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9523                         if (ret) {
9524                                 dev_info(&pf->pdev->dev,
9525                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9526                                          enabled_tc,
9527                                          i40e_stat_str(&pf->hw, ret),
9528                                          i40e_aq_str(&pf->hw,
9529                                                     pf->hw.aq.asq_last_status));
9530                                 ret = -ENOENT;
9531                         }
9532                 }
9533                 break;
9534
9535         case I40E_VSI_FDIR:
9536                 ctxt.pf_num = hw->pf_id;
9537                 ctxt.vf_num = 0;
9538                 ctxt.uplink_seid = vsi->uplink_seid;
9539                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9540                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9541                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9542                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
9543                         ctxt.info.valid_sections |=
9544                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9545                         ctxt.info.switch_id =
9546                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9547                 }
9548                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9549                 break;
9550
9551         case I40E_VSI_VMDQ2:
9552                 ctxt.pf_num = hw->pf_id;
9553                 ctxt.vf_num = 0;
9554                 ctxt.uplink_seid = vsi->uplink_seid;
9555                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9556                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9557
9558                 /* This VSI is connected to VEB so the switch_id
9559                  * should be set to zero by default.
9560                  */
9561                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9562                         ctxt.info.valid_sections |=
9563                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9564                         ctxt.info.switch_id =
9565                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9566                 }
9567
9568                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9569                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9570                 break;
9571
9572         case I40E_VSI_SRIOV:
9573                 ctxt.pf_num = hw->pf_id;
9574                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9575                 ctxt.uplink_seid = vsi->uplink_seid;
9576                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9577                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9578
9579                 /* This VSI is connected to VEB so the switch_id
9580                  * should be set to zero by default.
9581                  */
9582                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9583                         ctxt.info.valid_sections |=
9584                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9585                         ctxt.info.switch_id =
9586                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9587                 }
9588
9589                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9590                         ctxt.info.valid_sections |=
9591                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9592                         ctxt.info.queueing_opt_flags |=
9593                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
9594                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
9595                 }
9596
9597                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9598                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9599                 if (pf->vf[vsi->vf_id].spoofchk) {
9600                         ctxt.info.valid_sections |=
9601                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9602                         ctxt.info.sec_flags |=
9603                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9604                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9605                 }
9606                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9607                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9608                 break;
9609
9610         case I40E_VSI_IWARP:
9611                 /* send down message to iWARP */
9612                 break;
9613
9614         default:
9615                 return -ENODEV;
9616         }
9617
9618         if (vsi->type != I40E_VSI_MAIN) {
9619                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9620                 if (ret) {
9621                         dev_info(&vsi->back->pdev->dev,
9622                                  "add vsi failed, err %s aq_err %s\n",
9623                                  i40e_stat_str(&pf->hw, ret),
9624                                  i40e_aq_str(&pf->hw,
9625                                              pf->hw.aq.asq_last_status));
9626                         ret = -ENOENT;
9627                         goto err;
9628                 }
9629                 vsi->info = ctxt.info;
9630                 vsi->info.valid_sections = 0;
9631                 vsi->seid = ctxt.seid;
9632                 vsi->id = ctxt.vsi_number;
9633         }
9634
9635         vsi->active_filters = 0;
9636         clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
9637         spin_lock_bh(&vsi->mac_filter_hash_lock);
9638         /* If macvlan filters already exist, force them to get loaded */
9639         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
9640                 f->state = I40E_FILTER_NEW;
9641                 f_count++;
9642         }
9643         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9644
9645         if (f_count) {
9646                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
9647                 pf->flags |= I40E_FLAG_FILTER_SYNC;
9648         }
9649
9650         /* Update VSI BW information */
9651         ret = i40e_vsi_get_bw_info(vsi);
9652         if (ret) {
9653                 dev_info(&pf->pdev->dev,
9654                          "couldn't get vsi bw info, err %s aq_err %s\n",
9655                          i40e_stat_str(&pf->hw, ret),
9656                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9657                 /* VSI is already added so not tearing that up */
9658                 ret = 0;
9659         }
9660
9661 err:
9662         return ret;
9663 }
9664
9665 /**
9666  * i40e_vsi_release - Delete a VSI and free its resources
9667  * @vsi: the VSI being removed
9668  *
9669  * Returns 0 on success or < 0 on error
9670  **/
9671 int i40e_vsi_release(struct i40e_vsi *vsi)
9672 {
9673         struct i40e_mac_filter *f;
9674         struct hlist_node *h;
9675         struct i40e_veb *veb = NULL;
9676         struct i40e_pf *pf;
9677         u16 uplink_seid;
9678         int i, n, bkt;
9679
9680         pf = vsi->back;
9681
9682         /* release of a VEB-owner or last VSI is not allowed */
9683         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
9684                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
9685                          vsi->seid, vsi->uplink_seid);
9686                 return -ENODEV;
9687         }
9688         if (vsi == pf->vsi[pf->lan_vsi] &&
9689             !test_bit(__I40E_DOWN, &pf->state)) {
9690                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
9691                 return -ENODEV;
9692         }
9693
9694         uplink_seid = vsi->uplink_seid;
9695         if (vsi->type != I40E_VSI_SRIOV) {
9696                 if (vsi->netdev_registered) {
9697                         vsi->netdev_registered = false;
9698                         if (vsi->netdev) {
9699                                 /* results in a call to i40e_close() */
9700                                 unregister_netdev(vsi->netdev);
9701                         }
9702                 } else {
9703                         i40e_vsi_close(vsi);
9704                 }
9705                 i40e_vsi_disable_irq(vsi);
9706         }
9707
9708         spin_lock_bh(&vsi->mac_filter_hash_lock);
9709
9710         /* clear the sync flag on all filters */
9711         if (vsi->netdev) {
9712                 __dev_uc_unsync(vsi->netdev, NULL);
9713                 __dev_mc_unsync(vsi->netdev, NULL);
9714         }
9715
9716         /* make sure any remaining filters are marked for deletion */
9717         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
9718                 __i40e_del_filter(vsi, f);
9719
9720         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9721
9722         i40e_sync_vsi_filters(vsi);
9723
9724         i40e_vsi_delete(vsi);
9725         i40e_vsi_free_q_vectors(vsi);
9726         if (vsi->netdev) {
9727                 free_netdev(vsi->netdev);
9728                 vsi->netdev = NULL;
9729         }
9730         i40e_vsi_clear_rings(vsi);
9731         i40e_vsi_clear(vsi);
9732
9733         /* If this was the last thing on the VEB, except for the
9734          * controlling VSI, remove the VEB, which puts the controlling
9735          * VSI onto the next level down in the switch.
9736          *
9737          * Well, okay, there's one more exception here: don't remove
9738          * the orphan VEBs yet.  We'll wait for an explicit remove request
9739          * from up the network stack.
9740          */
9741         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
9742                 if (pf->vsi[i] &&
9743                     pf->vsi[i]->uplink_seid == uplink_seid &&
9744                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
9745                         n++;      /* count the VSIs */
9746                 }
9747         }
9748         for (i = 0; i < I40E_MAX_VEB; i++) {
9749                 if (!pf->veb[i])
9750                         continue;
9751                 if (pf->veb[i]->uplink_seid == uplink_seid)
9752                         n++;     /* count the VEBs */
9753                 if (pf->veb[i]->seid == uplink_seid)
9754                         veb = pf->veb[i];
9755         }
9756         if (n == 0 && veb && veb->uplink_seid != 0)
9757                 i40e_veb_release(veb);
9758
9759         return 0;
9760 }
9761
9762 /**
9763  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
9764  * @vsi: ptr to the VSI
9765  *
9766  * This should only be called after i40e_vsi_mem_alloc() which allocates the
9767  * corresponding SW VSI structure and initializes num_queue_pairs for the
9768  * newly allocated VSI.
9769  *
9770  * Returns 0 on success or negative on failure
9771  **/
9772 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
9773 {
9774         int ret = -ENOENT;
9775         struct i40e_pf *pf = vsi->back;
9776
9777         if (vsi->q_vectors[0]) {
9778                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
9779                          vsi->seid);
9780                 return -EEXIST;
9781         }
9782
9783         if (vsi->base_vector) {
9784                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
9785                          vsi->seid, vsi->base_vector);
9786                 return -EEXIST;
9787         }
9788
9789         ret = i40e_vsi_alloc_q_vectors(vsi);
9790         if (ret) {
9791                 dev_info(&pf->pdev->dev,
9792                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
9793                          vsi->num_q_vectors, vsi->seid, ret);
9794                 vsi->num_q_vectors = 0;
9795                 goto vector_setup_out;
9796         }
9797
9798         /* In Legacy mode, we do not have to get any other vector since we
9799          * piggyback on the misc/ICR0 for queue interrupts.
9800         */
9801         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
9802                 return ret;
9803         if (vsi->num_q_vectors)
9804                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
9805                                                  vsi->num_q_vectors, vsi->idx);
9806         if (vsi->base_vector < 0) {
9807                 dev_info(&pf->pdev->dev,
9808                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
9809                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
9810                 i40e_vsi_free_q_vectors(vsi);
9811                 ret = -ENOENT;
9812                 goto vector_setup_out;
9813         }
9814
9815 vector_setup_out:
9816         return ret;
9817 }
9818
9819 /**
9820  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
9821  * @vsi: pointer to the vsi.
9822  *
9823  * This re-allocates a vsi's queue resources.
9824  *
9825  * Returns pointer to the successfully allocated and configured VSI sw struct
9826  * on success, otherwise returns NULL on failure.
9827  **/
9828 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
9829 {
9830         struct i40e_pf *pf;
9831         u8 enabled_tc;
9832         int ret;
9833
9834         if (!vsi)
9835                 return NULL;
9836
9837         pf = vsi->back;
9838
9839         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
9840         i40e_vsi_clear_rings(vsi);
9841
9842         i40e_vsi_free_arrays(vsi, false);
9843         i40e_set_num_rings_in_vsi(vsi);
9844         ret = i40e_vsi_alloc_arrays(vsi, false);
9845         if (ret)
9846                 goto err_vsi;
9847
9848         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
9849         if (ret < 0) {
9850                 dev_info(&pf->pdev->dev,
9851                          "failed to get tracking for %d queues for VSI %d err %d\n",
9852                          vsi->alloc_queue_pairs, vsi->seid, ret);
9853                 goto err_vsi;
9854         }
9855         vsi->base_queue = ret;
9856
9857         /* Update the FW view of the VSI. Force a reset of TC and queue
9858          * layout configurations.
9859          */
9860         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
9861         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
9862         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
9863         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
9864         if (vsi->type == I40E_VSI_MAIN)
9865                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
9866
9867         /* assign it some queues */
9868         ret = i40e_alloc_rings(vsi);
9869         if (ret)
9870                 goto err_rings;
9871
9872         /* map all of the rings to the q_vectors */
9873         i40e_vsi_map_rings_to_vectors(vsi);
9874         return vsi;
9875
9876 err_rings:
9877         i40e_vsi_free_q_vectors(vsi);
9878         if (vsi->netdev_registered) {
9879                 vsi->netdev_registered = false;
9880                 unregister_netdev(vsi->netdev);
9881                 free_netdev(vsi->netdev);
9882                 vsi->netdev = NULL;
9883         }
9884         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
9885 err_vsi:
9886         i40e_vsi_clear(vsi);
9887         return NULL;
9888 }
9889
9890 /**
9891  * i40e_vsi_setup - Set up a VSI by a given type
9892  * @pf: board private structure
9893  * @type: VSI type
9894  * @uplink_seid: the switch element to link to
9895  * @param1: usage depends upon VSI type. For VF types, indicates VF id
9896  *
9897  * This allocates the sw VSI structure and its queue resources, then add a VSI
9898  * to the identified VEB.
9899  *
9900  * Returns pointer to the successfully allocated and configure VSI sw struct on
9901  * success, otherwise returns NULL on failure.
9902  **/
9903 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
9904                                 u16 uplink_seid, u32 param1)
9905 {
9906         struct i40e_vsi *vsi = NULL;
9907         struct i40e_veb *veb = NULL;
9908         int ret, i;
9909         int v_idx;
9910
9911         /* The requested uplink_seid must be either
9912          *     - the PF's port seid
9913          *              no VEB is needed because this is the PF
9914          *              or this is a Flow Director special case VSI
9915          *     - seid of an existing VEB
9916          *     - seid of a VSI that owns an existing VEB
9917          *     - seid of a VSI that doesn't own a VEB
9918          *              a new VEB is created and the VSI becomes the owner
9919          *     - seid of the PF VSI, which is what creates the first VEB
9920          *              this is a special case of the previous
9921          *
9922          * Find which uplink_seid we were given and create a new VEB if needed
9923          */
9924         for (i = 0; i < I40E_MAX_VEB; i++) {
9925                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
9926                         veb = pf->veb[i];
9927                         break;
9928                 }
9929         }
9930
9931         if (!veb && uplink_seid != pf->mac_seid) {
9932
9933                 for (i = 0; i < pf->num_alloc_vsi; i++) {
9934                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
9935                                 vsi = pf->vsi[i];
9936                                 break;
9937                         }
9938                 }
9939                 if (!vsi) {
9940                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
9941                                  uplink_seid);
9942                         return NULL;
9943                 }
9944
9945                 if (vsi->uplink_seid == pf->mac_seid)
9946                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
9947                                              vsi->tc_config.enabled_tc);
9948                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
9949                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9950                                              vsi->tc_config.enabled_tc);
9951                 if (veb) {
9952                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
9953                                 dev_info(&vsi->back->pdev->dev,
9954                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
9955                                 return NULL;
9956                         }
9957                         /* We come up by default in VEPA mode if SRIOV is not
9958                          * already enabled, in which case we can't force VEPA
9959                          * mode.
9960                          */
9961                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
9962                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
9963                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9964                         }
9965                         i40e_config_bridge_mode(veb);
9966                 }
9967                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9968                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9969                                 veb = pf->veb[i];
9970                 }
9971                 if (!veb) {
9972                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
9973                         return NULL;
9974                 }
9975
9976                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
9977                 uplink_seid = veb->seid;
9978         }
9979
9980         /* get vsi sw struct */
9981         v_idx = i40e_vsi_mem_alloc(pf, type);
9982         if (v_idx < 0)
9983                 goto err_alloc;
9984         vsi = pf->vsi[v_idx];
9985         if (!vsi)
9986                 goto err_alloc;
9987         vsi->type = type;
9988         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
9989
9990         if (type == I40E_VSI_MAIN)
9991                 pf->lan_vsi = v_idx;
9992         else if (type == I40E_VSI_SRIOV)
9993                 vsi->vf_id = param1;
9994         /* assign it some queues */
9995         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
9996                                 vsi->idx);
9997         if (ret < 0) {
9998                 dev_info(&pf->pdev->dev,
9999                          "failed to get tracking for %d queues for VSI %d err=%d\n",
10000                          vsi->alloc_queue_pairs, vsi->seid, ret);
10001                 goto err_vsi;
10002         }
10003         vsi->base_queue = ret;
10004
10005         /* get a VSI from the hardware */
10006         vsi->uplink_seid = uplink_seid;
10007         ret = i40e_add_vsi(vsi);
10008         if (ret)
10009                 goto err_vsi;
10010
10011         switch (vsi->type) {
10012         /* setup the netdev if needed */
10013         case I40E_VSI_MAIN:
10014                 /* Apply relevant filters if a platform-specific mac
10015                  * address was selected.
10016                  */
10017                 if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
10018                         ret = i40e_macaddr_init(vsi, pf->hw.mac.addr);
10019                         if (ret) {
10020                                 dev_warn(&pf->pdev->dev,
10021                                          "could not set up macaddr; err %d\n",
10022                                          ret);
10023                         }
10024                 }
10025         case I40E_VSI_VMDQ2:
10026                 ret = i40e_config_netdev(vsi);
10027                 if (ret)
10028                         goto err_netdev;
10029                 ret = register_netdev(vsi->netdev);
10030                 if (ret)
10031                         goto err_netdev;
10032                 vsi->netdev_registered = true;
10033                 netif_carrier_off(vsi->netdev);
10034 #ifdef CONFIG_I40E_DCB
10035                 /* Setup DCB netlink interface */
10036                 i40e_dcbnl_setup(vsi);
10037 #endif /* CONFIG_I40E_DCB */
10038                 /* fall through */
10039
10040         case I40E_VSI_FDIR:
10041                 /* set up vectors and rings if needed */
10042                 ret = i40e_vsi_setup_vectors(vsi);
10043                 if (ret)
10044                         goto err_msix;
10045
10046                 ret = i40e_alloc_rings(vsi);
10047                 if (ret)
10048                         goto err_rings;
10049
10050                 /* map all of the rings to the q_vectors */
10051                 i40e_vsi_map_rings_to_vectors(vsi);
10052
10053                 i40e_vsi_reset_stats(vsi);
10054                 break;
10055
10056         default:
10057                 /* no netdev or rings for the other VSI types */
10058                 break;
10059         }
10060
10061         if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
10062             (vsi->type == I40E_VSI_VMDQ2)) {
10063                 ret = i40e_vsi_config_rss(vsi);
10064         }
10065         return vsi;
10066
10067 err_rings:
10068         i40e_vsi_free_q_vectors(vsi);
10069 err_msix:
10070         if (vsi->netdev_registered) {
10071                 vsi->netdev_registered = false;
10072                 unregister_netdev(vsi->netdev);
10073                 free_netdev(vsi->netdev);
10074                 vsi->netdev = NULL;
10075         }
10076 err_netdev:
10077         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10078 err_vsi:
10079         i40e_vsi_clear(vsi);
10080 err_alloc:
10081         return NULL;
10082 }
10083
10084 /**
10085  * i40e_veb_get_bw_info - Query VEB BW information
10086  * @veb: the veb to query
10087  *
10088  * Query the Tx scheduler BW configuration data for given VEB
10089  **/
10090 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10091 {
10092         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10093         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10094         struct i40e_pf *pf = veb->pf;
10095         struct i40e_hw *hw = &pf->hw;
10096         u32 tc_bw_max;
10097         int ret = 0;
10098         int i;
10099
10100         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10101                                                   &bw_data, NULL);
10102         if (ret) {
10103                 dev_info(&pf->pdev->dev,
10104                          "query veb bw config failed, err %s aq_err %s\n",
10105                          i40e_stat_str(&pf->hw, ret),
10106                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10107                 goto out;
10108         }
10109
10110         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10111                                                    &ets_data, NULL);
10112         if (ret) {
10113                 dev_info(&pf->pdev->dev,
10114                          "query veb bw ets config failed, err %s aq_err %s\n",
10115                          i40e_stat_str(&pf->hw, ret),
10116                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10117                 goto out;
10118         }
10119
10120         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10121         veb->bw_max_quanta = ets_data.tc_bw_max;
10122         veb->is_abs_credits = bw_data.absolute_credits_enable;
10123         veb->enabled_tc = ets_data.tc_valid_bits;
10124         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10125                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10126         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10127                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10128                 veb->bw_tc_limit_credits[i] =
10129                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
10130                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10131         }
10132
10133 out:
10134         return ret;
10135 }
10136
10137 /**
10138  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10139  * @pf: board private structure
10140  *
10141  * On error: returns error code (negative)
10142  * On success: returns vsi index in PF (positive)
10143  **/
10144 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10145 {
10146         int ret = -ENOENT;
10147         struct i40e_veb *veb;
10148         int i;
10149
10150         /* Need to protect the allocation of switch elements at the PF level */
10151         mutex_lock(&pf->switch_mutex);
10152
10153         /* VEB list may be fragmented if VEB creation/destruction has
10154          * been happening.  We can afford to do a quick scan to look
10155          * for any free slots in the list.
10156          *
10157          * find next empty veb slot, looping back around if necessary
10158          */
10159         i = 0;
10160         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10161                 i++;
10162         if (i >= I40E_MAX_VEB) {
10163                 ret = -ENOMEM;
10164                 goto err_alloc_veb;  /* out of VEB slots! */
10165         }
10166
10167         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10168         if (!veb) {
10169                 ret = -ENOMEM;
10170                 goto err_alloc_veb;
10171         }
10172         veb->pf = pf;
10173         veb->idx = i;
10174         veb->enabled_tc = 1;
10175
10176         pf->veb[i] = veb;
10177         ret = i;
10178 err_alloc_veb:
10179         mutex_unlock(&pf->switch_mutex);
10180         return ret;
10181 }
10182
10183 /**
10184  * i40e_switch_branch_release - Delete a branch of the switch tree
10185  * @branch: where to start deleting
10186  *
10187  * This uses recursion to find the tips of the branch to be
10188  * removed, deleting until we get back to and can delete this VEB.
10189  **/
10190 static void i40e_switch_branch_release(struct i40e_veb *branch)
10191 {
10192         struct i40e_pf *pf = branch->pf;
10193         u16 branch_seid = branch->seid;
10194         u16 veb_idx = branch->idx;
10195         int i;
10196
10197         /* release any VEBs on this VEB - RECURSION */
10198         for (i = 0; i < I40E_MAX_VEB; i++) {
10199                 if (!pf->veb[i])
10200                         continue;
10201                 if (pf->veb[i]->uplink_seid == branch->seid)
10202                         i40e_switch_branch_release(pf->veb[i]);
10203         }
10204
10205         /* Release the VSIs on this VEB, but not the owner VSI.
10206          *
10207          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10208          *       the VEB itself, so don't use (*branch) after this loop.
10209          */
10210         for (i = 0; i < pf->num_alloc_vsi; i++) {
10211                 if (!pf->vsi[i])
10212                         continue;
10213                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10214                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10215                         i40e_vsi_release(pf->vsi[i]);
10216                 }
10217         }
10218
10219         /* There's one corner case where the VEB might not have been
10220          * removed, so double check it here and remove it if needed.
10221          * This case happens if the veb was created from the debugfs
10222          * commands and no VSIs were added to it.
10223          */
10224         if (pf->veb[veb_idx])
10225                 i40e_veb_release(pf->veb[veb_idx]);
10226 }
10227
10228 /**
10229  * i40e_veb_clear - remove veb struct
10230  * @veb: the veb to remove
10231  **/
10232 static void i40e_veb_clear(struct i40e_veb *veb)
10233 {
10234         if (!veb)
10235                 return;
10236
10237         if (veb->pf) {
10238                 struct i40e_pf *pf = veb->pf;
10239
10240                 mutex_lock(&pf->switch_mutex);
10241                 if (pf->veb[veb->idx] == veb)
10242                         pf->veb[veb->idx] = NULL;
10243                 mutex_unlock(&pf->switch_mutex);
10244         }
10245
10246         kfree(veb);
10247 }
10248
10249 /**
10250  * i40e_veb_release - Delete a VEB and free its resources
10251  * @veb: the VEB being removed
10252  **/
10253 void i40e_veb_release(struct i40e_veb *veb)
10254 {
10255         struct i40e_vsi *vsi = NULL;
10256         struct i40e_pf *pf;
10257         int i, n = 0;
10258
10259         pf = veb->pf;
10260
10261         /* find the remaining VSI and check for extras */
10262         for (i = 0; i < pf->num_alloc_vsi; i++) {
10263                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10264                         n++;
10265                         vsi = pf->vsi[i];
10266                 }
10267         }
10268         if (n != 1) {
10269                 dev_info(&pf->pdev->dev,
10270                          "can't remove VEB %d with %d VSIs left\n",
10271                          veb->seid, n);
10272                 return;
10273         }
10274
10275         /* move the remaining VSI to uplink veb */
10276         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10277         if (veb->uplink_seid) {
10278                 vsi->uplink_seid = veb->uplink_seid;
10279                 if (veb->uplink_seid == pf->mac_seid)
10280                         vsi->veb_idx = I40E_NO_VEB;
10281                 else
10282                         vsi->veb_idx = veb->veb_idx;
10283         } else {
10284                 /* floating VEB */
10285                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10286                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10287         }
10288
10289         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10290         i40e_veb_clear(veb);
10291 }
10292
10293 /**
10294  * i40e_add_veb - create the VEB in the switch
10295  * @veb: the VEB to be instantiated
10296  * @vsi: the controlling VSI
10297  **/
10298 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10299 {
10300         struct i40e_pf *pf = veb->pf;
10301         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10302         int ret;
10303
10304         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10305                               veb->enabled_tc, false,
10306                               &veb->seid, enable_stats, NULL);
10307
10308         /* get a VEB from the hardware */
10309         if (ret) {
10310                 dev_info(&pf->pdev->dev,
10311                          "couldn't add VEB, err %s aq_err %s\n",
10312                          i40e_stat_str(&pf->hw, ret),
10313                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10314                 return -EPERM;
10315         }
10316
10317         /* get statistics counter */
10318         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10319                                          &veb->stats_idx, NULL, NULL, NULL);
10320         if (ret) {
10321                 dev_info(&pf->pdev->dev,
10322                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10323                          i40e_stat_str(&pf->hw, ret),
10324                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10325                 return -EPERM;
10326         }
10327         ret = i40e_veb_get_bw_info(veb);
10328         if (ret) {
10329                 dev_info(&pf->pdev->dev,
10330                          "couldn't get VEB bw info, err %s aq_err %s\n",
10331                          i40e_stat_str(&pf->hw, ret),
10332                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10333                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10334                 return -ENOENT;
10335         }
10336
10337         vsi->uplink_seid = veb->seid;
10338         vsi->veb_idx = veb->idx;
10339         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10340
10341         return 0;
10342 }
10343
10344 /**
10345  * i40e_veb_setup - Set up a VEB
10346  * @pf: board private structure
10347  * @flags: VEB setup flags
10348  * @uplink_seid: the switch element to link to
10349  * @vsi_seid: the initial VSI seid
10350  * @enabled_tc: Enabled TC bit-map
10351  *
10352  * This allocates the sw VEB structure and links it into the switch
10353  * It is possible and legal for this to be a duplicate of an already
10354  * existing VEB.  It is also possible for both uplink and vsi seids
10355  * to be zero, in order to create a floating VEB.
10356  *
10357  * Returns pointer to the successfully allocated VEB sw struct on
10358  * success, otherwise returns NULL on failure.
10359  **/
10360 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10361                                 u16 uplink_seid, u16 vsi_seid,
10362                                 u8 enabled_tc)
10363 {
10364         struct i40e_veb *veb, *uplink_veb = NULL;
10365         int vsi_idx, veb_idx;
10366         int ret;
10367
10368         /* if one seid is 0, the other must be 0 to create a floating relay */
10369         if ((uplink_seid == 0 || vsi_seid == 0) &&
10370             (uplink_seid + vsi_seid != 0)) {
10371                 dev_info(&pf->pdev->dev,
10372                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10373                          uplink_seid, vsi_seid);
10374                 return NULL;
10375         }
10376
10377         /* make sure there is such a vsi and uplink */
10378         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10379                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10380                         break;
10381         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10382                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10383                          vsi_seid);
10384                 return NULL;
10385         }
10386
10387         if (uplink_seid && uplink_seid != pf->mac_seid) {
10388                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10389                         if (pf->veb[veb_idx] &&
10390                             pf->veb[veb_idx]->seid == uplink_seid) {
10391                                 uplink_veb = pf->veb[veb_idx];
10392                                 break;
10393                         }
10394                 }
10395                 if (!uplink_veb) {
10396                         dev_info(&pf->pdev->dev,
10397                                  "uplink seid %d not found\n", uplink_seid);
10398                         return NULL;
10399                 }
10400         }
10401
10402         /* get veb sw struct */
10403         veb_idx = i40e_veb_mem_alloc(pf);
10404         if (veb_idx < 0)
10405                 goto err_alloc;
10406         veb = pf->veb[veb_idx];
10407         veb->flags = flags;
10408         veb->uplink_seid = uplink_seid;
10409         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10410         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10411
10412         /* create the VEB in the switch */
10413         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10414         if (ret)
10415                 goto err_veb;
10416         if (vsi_idx == pf->lan_vsi)
10417                 pf->lan_veb = veb->idx;
10418
10419         return veb;
10420
10421 err_veb:
10422         i40e_veb_clear(veb);
10423 err_alloc:
10424         return NULL;
10425 }
10426
10427 /**
10428  * i40e_setup_pf_switch_element - set PF vars based on switch type
10429  * @pf: board private structure
10430  * @ele: element we are building info from
10431  * @num_reported: total number of elements
10432  * @printconfig: should we print the contents
10433  *
10434  * helper function to assist in extracting a few useful SEID values.
10435  **/
10436 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10437                                 struct i40e_aqc_switch_config_element_resp *ele,
10438                                 u16 num_reported, bool printconfig)
10439 {
10440         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10441         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10442         u8 element_type = ele->element_type;
10443         u16 seid = le16_to_cpu(ele->seid);
10444
10445         if (printconfig)
10446                 dev_info(&pf->pdev->dev,
10447                          "type=%d seid=%d uplink=%d downlink=%d\n",
10448                          element_type, seid, uplink_seid, downlink_seid);
10449
10450         switch (element_type) {
10451         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10452                 pf->mac_seid = seid;
10453                 break;
10454         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10455                 /* Main VEB? */
10456                 if (uplink_seid != pf->mac_seid)
10457                         break;
10458                 if (pf->lan_veb == I40E_NO_VEB) {
10459                         int v;
10460
10461                         /* find existing or else empty VEB */
10462                         for (v = 0; v < I40E_MAX_VEB; v++) {
10463                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10464                                         pf->lan_veb = v;
10465                                         break;
10466                                 }
10467                         }
10468                         if (pf->lan_veb == I40E_NO_VEB) {
10469                                 v = i40e_veb_mem_alloc(pf);
10470                                 if (v < 0)
10471                                         break;
10472                                 pf->lan_veb = v;
10473                         }
10474                 }
10475
10476                 pf->veb[pf->lan_veb]->seid = seid;
10477                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10478                 pf->veb[pf->lan_veb]->pf = pf;
10479                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10480                 break;
10481         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10482                 if (num_reported != 1)
10483                         break;
10484                 /* This is immediately after a reset so we can assume this is
10485                  * the PF's VSI
10486                  */
10487                 pf->mac_seid = uplink_seid;
10488                 pf->pf_seid = downlink_seid;
10489                 pf->main_vsi_seid = seid;
10490                 if (printconfig)
10491                         dev_info(&pf->pdev->dev,
10492                                  "pf_seid=%d main_vsi_seid=%d\n",
10493                                  pf->pf_seid, pf->main_vsi_seid);
10494                 break;
10495         case I40E_SWITCH_ELEMENT_TYPE_PF:
10496         case I40E_SWITCH_ELEMENT_TYPE_VF:
10497         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10498         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10499         case I40E_SWITCH_ELEMENT_TYPE_PE:
10500         case I40E_SWITCH_ELEMENT_TYPE_PA:
10501                 /* ignore these for now */
10502                 break;
10503         default:
10504                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10505                          element_type, seid);
10506                 break;
10507         }
10508 }
10509
10510 /**
10511  * i40e_fetch_switch_configuration - Get switch config from firmware
10512  * @pf: board private structure
10513  * @printconfig: should we print the contents
10514  *
10515  * Get the current switch configuration from the device and
10516  * extract a few useful SEID values.
10517  **/
10518 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10519 {
10520         struct i40e_aqc_get_switch_config_resp *sw_config;
10521         u16 next_seid = 0;
10522         int ret = 0;
10523         u8 *aq_buf;
10524         int i;
10525
10526         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10527         if (!aq_buf)
10528                 return -ENOMEM;
10529
10530         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10531         do {
10532                 u16 num_reported, num_total;
10533
10534                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10535                                                 I40E_AQ_LARGE_BUF,
10536                                                 &next_seid, NULL);
10537                 if (ret) {
10538                         dev_info(&pf->pdev->dev,
10539                                  "get switch config failed err %s aq_err %s\n",
10540                                  i40e_stat_str(&pf->hw, ret),
10541                                  i40e_aq_str(&pf->hw,
10542                                              pf->hw.aq.asq_last_status));
10543                         kfree(aq_buf);
10544                         return -ENOENT;
10545                 }
10546
10547                 num_reported = le16_to_cpu(sw_config->header.num_reported);
10548                 num_total = le16_to_cpu(sw_config->header.num_total);
10549
10550                 if (printconfig)
10551                         dev_info(&pf->pdev->dev,
10552                                  "header: %d reported %d total\n",
10553                                  num_reported, num_total);
10554
10555                 for (i = 0; i < num_reported; i++) {
10556                         struct i40e_aqc_switch_config_element_resp *ele =
10557                                 &sw_config->element[i];
10558
10559                         i40e_setup_pf_switch_element(pf, ele, num_reported,
10560                                                      printconfig);
10561                 }
10562         } while (next_seid != 0);
10563
10564         kfree(aq_buf);
10565         return ret;
10566 }
10567
10568 /**
10569  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10570  * @pf: board private structure
10571  * @reinit: if the Main VSI needs to re-initialized.
10572  *
10573  * Returns 0 on success, negative value on failure
10574  **/
10575 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10576 {
10577         u16 flags = 0;
10578         int ret;
10579
10580         /* find out what's out there already */
10581         ret = i40e_fetch_switch_configuration(pf, false);
10582         if (ret) {
10583                 dev_info(&pf->pdev->dev,
10584                          "couldn't fetch switch config, err %s aq_err %s\n",
10585                          i40e_stat_str(&pf->hw, ret),
10586                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10587                 return ret;
10588         }
10589         i40e_pf_reset_stats(pf);
10590
10591         /* set the switch config bit for the whole device to
10592          * support limited promisc or true promisc
10593          * when user requests promisc. The default is limited
10594          * promisc.
10595         */
10596
10597         if ((pf->hw.pf_id == 0) &&
10598             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
10599                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10600
10601         if (pf->hw.pf_id == 0) {
10602                 u16 valid_flags;
10603
10604                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10605                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
10606                                                 NULL);
10607                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
10608                         dev_info(&pf->pdev->dev,
10609                                  "couldn't set switch config bits, err %s aq_err %s\n",
10610                                  i40e_stat_str(&pf->hw, ret),
10611                                  i40e_aq_str(&pf->hw,
10612                                              pf->hw.aq.asq_last_status));
10613                         /* not a fatal problem, just keep going */
10614                 }
10615         }
10616
10617         /* first time setup */
10618         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10619                 struct i40e_vsi *vsi = NULL;
10620                 u16 uplink_seid;
10621
10622                 /* Set up the PF VSI associated with the PF's main VSI
10623                  * that is already in the HW switch
10624                  */
10625                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10626                         uplink_seid = pf->veb[pf->lan_veb]->seid;
10627                 else
10628                         uplink_seid = pf->mac_seid;
10629                 if (pf->lan_vsi == I40E_NO_VSI)
10630                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10631                 else if (reinit)
10632                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10633                 if (!vsi) {
10634                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10635                         i40e_fdir_teardown(pf);
10636                         return -EAGAIN;
10637                 }
10638         } else {
10639                 /* force a reset of TC and queue layout configurations */
10640                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10641
10642                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10643                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10644                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10645         }
10646         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
10647
10648         i40e_fdir_sb_setup(pf);
10649
10650         /* Setup static PF queue filter control settings */
10651         ret = i40e_setup_pf_filter_control(pf);
10652         if (ret) {
10653                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
10654                          ret);
10655                 /* Failure here should not stop continuing other steps */
10656         }
10657
10658         /* enable RSS in the HW, even for only one queue, as the stack can use
10659          * the hash
10660          */
10661         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
10662                 i40e_pf_config_rss(pf);
10663
10664         /* fill in link information and enable LSE reporting */
10665         i40e_link_event(pf);
10666
10667         /* Initialize user-specific link properties */
10668         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
10669                                   I40E_AQ_AN_COMPLETED) ? true : false);
10670
10671         i40e_ptp_init(pf);
10672
10673         return ret;
10674 }
10675
10676 /**
10677  * i40e_determine_queue_usage - Work out queue distribution
10678  * @pf: board private structure
10679  **/
10680 static void i40e_determine_queue_usage(struct i40e_pf *pf)
10681 {
10682         int queues_left;
10683
10684         pf->num_lan_qps = 0;
10685
10686         /* Find the max queues to be put into basic use.  We'll always be
10687          * using TC0, whether or not DCB is running, and TC0 will get the
10688          * big RSS set.
10689          */
10690         queues_left = pf->hw.func_caps.num_tx_qp;
10691
10692         if ((queues_left == 1) ||
10693             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
10694                 /* one qp for PF, no queues for anything else */
10695                 queues_left = 0;
10696                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10697
10698                 /* make sure all the fancies are disabled */
10699                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10700                                I40E_FLAG_IWARP_ENABLED  |
10701                                I40E_FLAG_FD_SB_ENABLED  |
10702                                I40E_FLAG_FD_ATR_ENABLED |
10703                                I40E_FLAG_DCB_CAPABLE    |
10704                                I40E_FLAG_DCB_ENABLED    |
10705                                I40E_FLAG_SRIOV_ENABLED  |
10706                                I40E_FLAG_VMDQ_ENABLED);
10707         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
10708                                   I40E_FLAG_FD_SB_ENABLED |
10709                                   I40E_FLAG_FD_ATR_ENABLED |
10710                                   I40E_FLAG_DCB_CAPABLE))) {
10711                 /* one qp for PF */
10712                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10713                 queues_left -= pf->num_lan_qps;
10714
10715                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10716                                I40E_FLAG_IWARP_ENABLED  |
10717                                I40E_FLAG_FD_SB_ENABLED  |
10718                                I40E_FLAG_FD_ATR_ENABLED |
10719                                I40E_FLAG_DCB_ENABLED    |
10720                                I40E_FLAG_VMDQ_ENABLED);
10721         } else {
10722                 /* Not enough queues for all TCs */
10723                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
10724                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
10725                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
10726                                         I40E_FLAG_DCB_ENABLED);
10727                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
10728                 }
10729                 pf->num_lan_qps = max_t(int, pf->rss_size_max,
10730                                         num_online_cpus());
10731                 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
10732                                         pf->hw.func_caps.num_tx_qp);
10733
10734                 queues_left -= pf->num_lan_qps;
10735         }
10736
10737         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10738                 if (queues_left > 1) {
10739                         queues_left -= 1; /* save 1 queue for FD */
10740                 } else {
10741                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10742                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
10743                 }
10744         }
10745
10746         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
10747             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
10748                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
10749                                         (queues_left / pf->num_vf_qps));
10750                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
10751         }
10752
10753         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
10754             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
10755                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
10756                                           (queues_left / pf->num_vmdq_qps));
10757                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
10758         }
10759
10760         pf->queues_left = queues_left;
10761         dev_dbg(&pf->pdev->dev,
10762                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
10763                 pf->hw.func_caps.num_tx_qp,
10764                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
10765                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
10766                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
10767                 queues_left);
10768 }
10769
10770 /**
10771  * i40e_setup_pf_filter_control - Setup PF static filter control
10772  * @pf: PF to be setup
10773  *
10774  * i40e_setup_pf_filter_control sets up a PF's initial filter control
10775  * settings. If PE/FCoE are enabled then it will also set the per PF
10776  * based filter sizes required for them. It also enables Flow director,
10777  * ethertype and macvlan type filter settings for the pf.
10778  *
10779  * Returns 0 on success, negative on failure
10780  **/
10781 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
10782 {
10783         struct i40e_filter_control_settings *settings = &pf->filter_settings;
10784
10785         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
10786
10787         /* Flow Director is enabled */
10788         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
10789                 settings->enable_fdir = true;
10790
10791         /* Ethtype and MACVLAN filters enabled for PF */
10792         settings->enable_ethtype = true;
10793         settings->enable_macvlan = true;
10794
10795         if (i40e_set_filter_control(&pf->hw, settings))
10796                 return -ENOENT;
10797
10798         return 0;
10799 }
10800
10801 #define INFO_STRING_LEN 255
10802 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
10803 static void i40e_print_features(struct i40e_pf *pf)
10804 {
10805         struct i40e_hw *hw = &pf->hw;
10806         char *buf;
10807         int i;
10808
10809         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
10810         if (!buf)
10811                 return;
10812
10813         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
10814 #ifdef CONFIG_PCI_IOV
10815         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
10816 #endif
10817         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
10818                       pf->hw.func_caps.num_vsis,
10819                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
10820         if (pf->flags & I40E_FLAG_RSS_ENABLED)
10821                 i += snprintf(&buf[i], REMAIN(i), " RSS");
10822         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
10823                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
10824         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10825                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
10826                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
10827         }
10828         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
10829                 i += snprintf(&buf[i], REMAIN(i), " DCB");
10830         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
10831         i += snprintf(&buf[i], REMAIN(i), " Geneve");
10832         if (pf->flags & I40E_FLAG_PTP)
10833                 i += snprintf(&buf[i], REMAIN(i), " PTP");
10834         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10835                 i += snprintf(&buf[i], REMAIN(i), " VEB");
10836         else
10837                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
10838
10839         dev_info(&pf->pdev->dev, "%s\n", buf);
10840         kfree(buf);
10841         WARN_ON(i > INFO_STRING_LEN);
10842 }
10843
10844 /**
10845  * i40e_get_platform_mac_addr - get platform-specific MAC address
10846  * @pdev: PCI device information struct
10847  * @pf: board private structure
10848  *
10849  * Look up the MAC address for the device. First we'll try
10850  * eth_platform_get_mac_address, which will check Open Firmware, or arch
10851  * specific fallback. Otherwise, we'll default to the stored value in
10852  * firmware.
10853  **/
10854 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
10855 {
10856         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
10857                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
10858 }
10859
10860 /**
10861  * i40e_probe - Device initialization routine
10862  * @pdev: PCI device information struct
10863  * @ent: entry in i40e_pci_tbl
10864  *
10865  * i40e_probe initializes a PF identified by a pci_dev structure.
10866  * The OS initialization, configuring of the PF private structure,
10867  * and a hardware reset occur.
10868  *
10869  * Returns 0 on success, negative on failure
10870  **/
10871 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
10872 {
10873         struct i40e_aq_get_phy_abilities_resp abilities;
10874         struct i40e_pf *pf;
10875         struct i40e_hw *hw;
10876         static u16 pfs_found;
10877         u16 wol_nvm_bits;
10878         u16 link_status;
10879         int err;
10880         u32 val;
10881         u32 i;
10882         u8 set_fc_aq_fail;
10883
10884         err = pci_enable_device_mem(pdev);
10885         if (err)
10886                 return err;
10887
10888         /* set up for high or low dma */
10889         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
10890         if (err) {
10891                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
10892                 if (err) {
10893                         dev_err(&pdev->dev,
10894                                 "DMA configuration failed: 0x%x\n", err);
10895                         goto err_dma;
10896                 }
10897         }
10898
10899         /* set up pci connections */
10900         err = pci_request_mem_regions(pdev, i40e_driver_name);
10901         if (err) {
10902                 dev_info(&pdev->dev,
10903                          "pci_request_selected_regions failed %d\n", err);
10904                 goto err_pci_reg;
10905         }
10906
10907         pci_enable_pcie_error_reporting(pdev);
10908         pci_set_master(pdev);
10909
10910         /* Now that we have a PCI connection, we need to do the
10911          * low level device setup.  This is primarily setting up
10912          * the Admin Queue structures and then querying for the
10913          * device's current profile information.
10914          */
10915         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
10916         if (!pf) {
10917                 err = -ENOMEM;
10918                 goto err_pf_alloc;
10919         }
10920         pf->next_vsi = 0;
10921         pf->pdev = pdev;
10922         set_bit(__I40E_DOWN, &pf->state);
10923
10924         hw = &pf->hw;
10925         hw->back = pf;
10926
10927         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
10928                                 I40E_MAX_CSR_SPACE);
10929
10930         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
10931         if (!hw->hw_addr) {
10932                 err = -EIO;
10933                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
10934                          (unsigned int)pci_resource_start(pdev, 0),
10935                          pf->ioremap_len, err);
10936                 goto err_ioremap;
10937         }
10938         hw->vendor_id = pdev->vendor;
10939         hw->device_id = pdev->device;
10940         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
10941         hw->subsystem_vendor_id = pdev->subsystem_vendor;
10942         hw->subsystem_device_id = pdev->subsystem_device;
10943         hw->bus.device = PCI_SLOT(pdev->devfn);
10944         hw->bus.func = PCI_FUNC(pdev->devfn);
10945         hw->bus.bus_id = pdev->bus->number;
10946         pf->instance = pfs_found;
10947
10948         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
10949         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
10950
10951         /* set up the locks for the AQ, do this only once in probe
10952          * and destroy them only once in remove
10953          */
10954         mutex_init(&hw->aq.asq_mutex);
10955         mutex_init(&hw->aq.arq_mutex);
10956
10957         pf->msg_enable = netif_msg_init(debug,
10958                                         NETIF_MSG_DRV |
10959                                         NETIF_MSG_PROBE |
10960                                         NETIF_MSG_LINK);
10961         if (debug < -1)
10962                 pf->hw.debug_mask = debug;
10963
10964         /* do a special CORER for clearing PXE mode once at init */
10965         if (hw->revision_id == 0 &&
10966             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
10967                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
10968                 i40e_flush(hw);
10969                 msleep(200);
10970                 pf->corer_count++;
10971
10972                 i40e_clear_pxe_mode(hw);
10973         }
10974
10975         /* Reset here to make sure all is clean and to define PF 'n' */
10976         i40e_clear_hw(hw);
10977         err = i40e_pf_reset(hw);
10978         if (err) {
10979                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
10980                 goto err_pf_reset;
10981         }
10982         pf->pfr_count++;
10983
10984         hw->aq.num_arq_entries = I40E_AQ_LEN;
10985         hw->aq.num_asq_entries = I40E_AQ_LEN;
10986         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
10987         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
10988         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
10989
10990         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
10991                  "%s-%s:misc",
10992                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
10993
10994         err = i40e_init_shared_code(hw);
10995         if (err) {
10996                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
10997                          err);
10998                 goto err_pf_reset;
10999         }
11000
11001         /* set up a default setting for link flow control */
11002         pf->hw.fc.requested_mode = I40E_FC_NONE;
11003
11004         err = i40e_init_adminq(hw);
11005         if (err) {
11006                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11007                         dev_info(&pdev->dev,
11008                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11009                 else
11010                         dev_info(&pdev->dev,
11011                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11012
11013                 goto err_pf_reset;
11014         }
11015
11016         /* provide nvm, fw, api versions */
11017         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11018                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11019                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
11020                  i40e_nvm_version_str(hw));
11021
11022         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11023             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
11024                 dev_info(&pdev->dev,
11025                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11026         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
11027                  hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
11028                 dev_info(&pdev->dev,
11029                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11030
11031         i40e_verify_eeprom(pf);
11032
11033         /* Rev 0 hardware was never productized */
11034         if (hw->revision_id < 1)
11035                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11036
11037         i40e_clear_pxe_mode(hw);
11038         err = i40e_get_capabilities(pf);
11039         if (err)
11040                 goto err_adminq_setup;
11041
11042         err = i40e_sw_init(pf);
11043         if (err) {
11044                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11045                 goto err_sw_init;
11046         }
11047
11048         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11049                                 hw->func_caps.num_rx_qp, 0, 0);
11050         if (err) {
11051                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11052                 goto err_init_lan_hmc;
11053         }
11054
11055         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11056         if (err) {
11057                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11058                 err = -ENOENT;
11059                 goto err_configure_lan_hmc;
11060         }
11061
11062         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11063          * Ignore error return codes because if it was already disabled via
11064          * hardware settings this will fail
11065          */
11066         if (pf->flags & I40E_FLAG_STOP_FW_LLDP) {
11067                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11068                 i40e_aq_stop_lldp(hw, true, NULL);
11069         }
11070
11071         /* allow a platform config to override the HW addr */
11072         i40e_get_platform_mac_addr(pdev, pf);
11073
11074         if (!is_valid_ether_addr(hw->mac.addr)) {
11075                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11076                 err = -EIO;
11077                 goto err_mac_addr;
11078         }
11079         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11080         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11081         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11082         if (is_valid_ether_addr(hw->mac.port_addr))
11083                 pf->flags |= I40E_FLAG_PORT_ID_VALID;
11084
11085         pci_set_drvdata(pdev, pf);
11086         pci_save_state(pdev);
11087 #ifdef CONFIG_I40E_DCB
11088         err = i40e_init_pf_dcb(pf);
11089         if (err) {
11090                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11091                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11092                 /* Continue without DCB enabled */
11093         }
11094 #endif /* CONFIG_I40E_DCB */
11095
11096         /* set up periodic task facility */
11097         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11098         pf->service_timer_period = HZ;
11099
11100         INIT_WORK(&pf->service_task, i40e_service_task);
11101         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
11102
11103         /* NVM bit on means WoL disabled for the port */
11104         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11105         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11106                 pf->wol_en = false;
11107         else
11108                 pf->wol_en = true;
11109         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11110
11111         /* set up the main switch operations */
11112         i40e_determine_queue_usage(pf);
11113         err = i40e_init_interrupt_scheme(pf);
11114         if (err)
11115                 goto err_switch_setup;
11116
11117         /* The number of VSIs reported by the FW is the minimum guaranteed
11118          * to us; HW supports far more and we share the remaining pool with
11119          * the other PFs. We allocate space for more than the guarantee with
11120          * the understanding that we might not get them all later.
11121          */
11122         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11123                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11124         else
11125                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11126
11127         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11128         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11129                           GFP_KERNEL);
11130         if (!pf->vsi) {
11131                 err = -ENOMEM;
11132                 goto err_switch_setup;
11133         }
11134
11135 #ifdef CONFIG_PCI_IOV
11136         /* prep for VF support */
11137         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11138             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11139             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11140                 if (pci_num_vf(pdev))
11141                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11142         }
11143 #endif
11144         err = i40e_setup_pf_switch(pf, false);
11145         if (err) {
11146                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11147                 goto err_vsis;
11148         }
11149
11150         /* Make sure flow control is set according to current settings */
11151         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11152         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11153                 dev_dbg(&pf->pdev->dev,
11154                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11155                         i40e_stat_str(hw, err),
11156                         i40e_aq_str(hw, hw->aq.asq_last_status));
11157         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11158                 dev_dbg(&pf->pdev->dev,
11159                         "Set fc with err %s aq_err %s on set_phy_config\n",
11160                         i40e_stat_str(hw, err),
11161                         i40e_aq_str(hw, hw->aq.asq_last_status));
11162         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11163                 dev_dbg(&pf->pdev->dev,
11164                         "Set fc with err %s aq_err %s on get_link_info\n",
11165                         i40e_stat_str(hw, err),
11166                         i40e_aq_str(hw, hw->aq.asq_last_status));
11167
11168         /* if FDIR VSI was set up, start it now */
11169         for (i = 0; i < pf->num_alloc_vsi; i++) {
11170                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11171                         i40e_vsi_open(pf->vsi[i]);
11172                         break;
11173                 }
11174         }
11175
11176         /* The driver only wants link up/down and module qualification
11177          * reports from firmware.  Note the negative logic.
11178          */
11179         err = i40e_aq_set_phy_int_mask(&pf->hw,
11180                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11181                                          I40E_AQ_EVENT_MEDIA_NA |
11182                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11183         if (err)
11184                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11185                          i40e_stat_str(&pf->hw, err),
11186                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11187
11188         /* Reconfigure hardware for allowing smaller MSS in the case
11189          * of TSO, so that we avoid the MDD being fired and causing
11190          * a reset in the case of small MSS+TSO.
11191          */
11192         val = rd32(hw, I40E_REG_MSS);
11193         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11194                 val &= ~I40E_REG_MSS_MIN_MASK;
11195                 val |= I40E_64BYTE_MSS;
11196                 wr32(hw, I40E_REG_MSS, val);
11197         }
11198
11199         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
11200                 msleep(75);
11201                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11202                 if (err)
11203                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11204                                  i40e_stat_str(&pf->hw, err),
11205                                  i40e_aq_str(&pf->hw,
11206                                              pf->hw.aq.asq_last_status));
11207         }
11208         /* The main driver is (mostly) up and happy. We need to set this state
11209          * before setting up the misc vector or we get a race and the vector
11210          * ends up disabled forever.
11211          */
11212         clear_bit(__I40E_DOWN, &pf->state);
11213
11214         /* In case of MSIX we are going to setup the misc vector right here
11215          * to handle admin queue events etc. In case of legacy and MSI
11216          * the misc functionality and queue processing is combined in
11217          * the same vector and that gets setup at open.
11218          */
11219         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11220                 err = i40e_setup_misc_vector(pf);
11221                 if (err) {
11222                         dev_info(&pdev->dev,
11223                                  "setup of misc vector failed: %d\n", err);
11224                         goto err_vsis;
11225                 }
11226         }
11227
11228 #ifdef CONFIG_PCI_IOV
11229         /* prep for VF support */
11230         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11231             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11232             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11233                 /* disable link interrupts for VFs */
11234                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11235                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11236                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11237                 i40e_flush(hw);
11238
11239                 if (pci_num_vf(pdev)) {
11240                         dev_info(&pdev->dev,
11241                                  "Active VFs found, allocating resources.\n");
11242                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11243                         if (err)
11244                                 dev_info(&pdev->dev,
11245                                          "Error %d allocating resources for existing VFs\n",
11246                                          err);
11247                 }
11248         }
11249 #endif /* CONFIG_PCI_IOV */
11250
11251         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11252                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11253                                                       pf->num_iwarp_msix,
11254                                                       I40E_IWARP_IRQ_PILE_ID);
11255                 if (pf->iwarp_base_vector < 0) {
11256                         dev_info(&pdev->dev,
11257                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11258                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11259                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11260                 }
11261         }
11262
11263         i40e_dbg_pf_init(pf);
11264
11265         /* tell the firmware that we're starting */
11266         i40e_send_version(pf);
11267
11268         /* since everything's happy, start the service_task timer */
11269         mod_timer(&pf->service_timer,
11270                   round_jiffies(jiffies + pf->service_timer_period));
11271
11272         /* add this PF to client device list and launch a client service task */
11273         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11274                 err = i40e_lan_add_device(pf);
11275                 if (err)
11276                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11277                                  err);
11278         }
11279
11280 #define PCI_SPEED_SIZE 8
11281 #define PCI_WIDTH_SIZE 8
11282         /* Devices on the IOSF bus do not have this information
11283          * and will report PCI Gen 1 x 1 by default so don't bother
11284          * checking them.
11285          */
11286         if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) {
11287                 char speed[PCI_SPEED_SIZE] = "Unknown";
11288                 char width[PCI_WIDTH_SIZE] = "Unknown";
11289
11290                 /* Get the negotiated link width and speed from PCI config
11291                  * space
11292                  */
11293                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11294                                           &link_status);
11295
11296                 i40e_set_pci_config_data(hw, link_status);
11297
11298                 switch (hw->bus.speed) {
11299                 case i40e_bus_speed_8000:
11300                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11301                 case i40e_bus_speed_5000:
11302                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11303                 case i40e_bus_speed_2500:
11304                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11305                 default:
11306                         break;
11307                 }
11308                 switch (hw->bus.width) {
11309                 case i40e_bus_width_pcie_x8:
11310                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11311                 case i40e_bus_width_pcie_x4:
11312                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11313                 case i40e_bus_width_pcie_x2:
11314                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11315                 case i40e_bus_width_pcie_x1:
11316                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11317                 default:
11318                         break;
11319                 }
11320
11321                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11322                          speed, width);
11323
11324                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11325                     hw->bus.speed < i40e_bus_speed_8000) {
11326                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11327                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11328                 }
11329         }
11330
11331         /* get the requested speeds from the fw */
11332         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11333         if (err)
11334                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11335                         i40e_stat_str(&pf->hw, err),
11336                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11337         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11338
11339         /* get the supported phy types from the fw */
11340         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11341         if (err)
11342                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11343                         i40e_stat_str(&pf->hw, err),
11344                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11345
11346         /* Add a filter to drop all Flow control frames from any VSI from being
11347          * transmitted. By doing so we stop a malicious VF from sending out
11348          * PAUSE or PFC frames and potentially controlling traffic for other
11349          * PF/VF VSIs.
11350          * The FW can still send Flow control frames if enabled.
11351          */
11352         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11353                                                        pf->main_vsi_seid);
11354
11355         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11356                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11357                 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS;
11358         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11359                 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER;
11360         /* print a string summarizing features */
11361         i40e_print_features(pf);
11362
11363         return 0;
11364
11365         /* Unwind what we've done if something failed in the setup */
11366 err_vsis:
11367         set_bit(__I40E_DOWN, &pf->state);
11368         i40e_clear_interrupt_scheme(pf);
11369         kfree(pf->vsi);
11370 err_switch_setup:
11371         i40e_reset_interrupt_capability(pf);
11372         del_timer_sync(&pf->service_timer);
11373 err_mac_addr:
11374 err_configure_lan_hmc:
11375         (void)i40e_shutdown_lan_hmc(hw);
11376 err_init_lan_hmc:
11377         kfree(pf->qp_pile);
11378 err_sw_init:
11379 err_adminq_setup:
11380 err_pf_reset:
11381         iounmap(hw->hw_addr);
11382 err_ioremap:
11383         kfree(pf);
11384 err_pf_alloc:
11385         pci_disable_pcie_error_reporting(pdev);
11386         pci_release_mem_regions(pdev);
11387 err_pci_reg:
11388 err_dma:
11389         pci_disable_device(pdev);
11390         return err;
11391 }
11392
11393 /**
11394  * i40e_remove - Device removal routine
11395  * @pdev: PCI device information struct
11396  *
11397  * i40e_remove is called by the PCI subsystem to alert the driver
11398  * that is should release a PCI device.  This could be caused by a
11399  * Hot-Plug event, or because the driver is going to be removed from
11400  * memory.
11401  **/
11402 static void i40e_remove(struct pci_dev *pdev)
11403 {
11404         struct i40e_pf *pf = pci_get_drvdata(pdev);
11405         struct i40e_hw *hw = &pf->hw;
11406         i40e_status ret_code;
11407         int i;
11408
11409         i40e_dbg_pf_exit(pf);
11410
11411         i40e_ptp_stop(pf);
11412
11413         /* Disable RSS in hw */
11414         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11415         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11416
11417         /* no more scheduling of any task */
11418         set_bit(__I40E_SUSPENDED, &pf->state);
11419         set_bit(__I40E_DOWN, &pf->state);
11420         if (pf->service_timer.data)
11421                 del_timer_sync(&pf->service_timer);
11422         if (pf->service_task.func)
11423                 cancel_work_sync(&pf->service_task);
11424
11425         /* Client close must be called explicitly here because the timer
11426          * has been stopped.
11427          */
11428         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11429
11430         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11431                 i40e_free_vfs(pf);
11432                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11433         }
11434
11435         i40e_fdir_teardown(pf);
11436
11437         /* If there is a switch structure or any orphans, remove them.
11438          * This will leave only the PF's VSI remaining.
11439          */
11440         for (i = 0; i < I40E_MAX_VEB; i++) {
11441                 if (!pf->veb[i])
11442                         continue;
11443
11444                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11445                     pf->veb[i]->uplink_seid == 0)
11446                         i40e_switch_branch_release(pf->veb[i]);
11447         }
11448
11449         /* Now we can shutdown the PF's VSI, just before we kill
11450          * adminq and hmc.
11451          */
11452         if (pf->vsi[pf->lan_vsi])
11453                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11454
11455         /* remove attached clients */
11456         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11457                 ret_code = i40e_lan_del_device(pf);
11458                 if (ret_code)
11459                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11460                                  ret_code);
11461         }
11462
11463         /* shutdown and destroy the HMC */
11464         if (hw->hmc.hmc_obj) {
11465                 ret_code = i40e_shutdown_lan_hmc(hw);
11466                 if (ret_code)
11467                         dev_warn(&pdev->dev,
11468                                  "Failed to destroy the HMC resources: %d\n",
11469                                  ret_code);
11470         }
11471
11472         /* shutdown the adminq */
11473         i40e_shutdown_adminq(hw);
11474
11475         /* destroy the locks only once, here */
11476         mutex_destroy(&hw->aq.arq_mutex);
11477         mutex_destroy(&hw->aq.asq_mutex);
11478
11479         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11480         i40e_clear_interrupt_scheme(pf);
11481         for (i = 0; i < pf->num_alloc_vsi; i++) {
11482                 if (pf->vsi[i]) {
11483                         i40e_vsi_clear_rings(pf->vsi[i]);
11484                         i40e_vsi_clear(pf->vsi[i]);
11485                         pf->vsi[i] = NULL;
11486                 }
11487         }
11488
11489         for (i = 0; i < I40E_MAX_VEB; i++) {
11490                 kfree(pf->veb[i]);
11491                 pf->veb[i] = NULL;
11492         }
11493
11494         kfree(pf->qp_pile);
11495         kfree(pf->vsi);
11496
11497         iounmap(hw->hw_addr);
11498         kfree(pf);
11499         pci_release_mem_regions(pdev);
11500
11501         pci_disable_pcie_error_reporting(pdev);
11502         pci_disable_device(pdev);
11503 }
11504
11505 /**
11506  * i40e_pci_error_detected - warning that something funky happened in PCI land
11507  * @pdev: PCI device information struct
11508  *
11509  * Called to warn that something happened and the error handling steps
11510  * are in progress.  Allows the driver to quiesce things, be ready for
11511  * remediation.
11512  **/
11513 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11514                                                 enum pci_channel_state error)
11515 {
11516         struct i40e_pf *pf = pci_get_drvdata(pdev);
11517
11518         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11519
11520         if (!pf) {
11521                 dev_info(&pdev->dev,
11522                          "Cannot recover - error happened during device probe\n");
11523                 return PCI_ERS_RESULT_DISCONNECT;
11524         }
11525
11526         /* shutdown all operations */
11527         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
11528                 rtnl_lock();
11529                 i40e_prep_for_reset(pf, true);
11530                 rtnl_unlock();
11531         }
11532
11533         /* Request a slot reset */
11534         return PCI_ERS_RESULT_NEED_RESET;
11535 }
11536
11537 /**
11538  * i40e_pci_error_slot_reset - a PCI slot reset just happened
11539  * @pdev: PCI device information struct
11540  *
11541  * Called to find if the driver can work with the device now that
11542  * the pci slot has been reset.  If a basic connection seems good
11543  * (registers are readable and have sane content) then return a
11544  * happy little PCI_ERS_RESULT_xxx.
11545  **/
11546 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11547 {
11548         struct i40e_pf *pf = pci_get_drvdata(pdev);
11549         pci_ers_result_t result;
11550         int err;
11551         u32 reg;
11552
11553         dev_dbg(&pdev->dev, "%s\n", __func__);
11554         if (pci_enable_device_mem(pdev)) {
11555                 dev_info(&pdev->dev,
11556                          "Cannot re-enable PCI device after reset.\n");
11557                 result = PCI_ERS_RESULT_DISCONNECT;
11558         } else {
11559                 pci_set_master(pdev);
11560                 pci_restore_state(pdev);
11561                 pci_save_state(pdev);
11562                 pci_wake_from_d3(pdev, false);
11563
11564                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11565                 if (reg == 0)
11566                         result = PCI_ERS_RESULT_RECOVERED;
11567                 else
11568                         result = PCI_ERS_RESULT_DISCONNECT;
11569         }
11570
11571         err = pci_cleanup_aer_uncorrect_error_status(pdev);
11572         if (err) {
11573                 dev_info(&pdev->dev,
11574                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11575                          err);
11576                 /* non-fatal, continue */
11577         }
11578
11579         return result;
11580 }
11581
11582 /**
11583  * i40e_pci_error_resume - restart operations after PCI error recovery
11584  * @pdev: PCI device information struct
11585  *
11586  * Called to allow the driver to bring things back up after PCI error
11587  * and/or reset recovery has finished.
11588  **/
11589 static void i40e_pci_error_resume(struct pci_dev *pdev)
11590 {
11591         struct i40e_pf *pf = pci_get_drvdata(pdev);
11592
11593         dev_dbg(&pdev->dev, "%s\n", __func__);
11594         if (test_bit(__I40E_SUSPENDED, &pf->state))
11595                 return;
11596
11597         rtnl_lock();
11598         i40e_handle_reset_warning(pf, true);
11599         rtnl_unlock();
11600 }
11601
11602 /**
11603  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
11604  * using the mac_address_write admin q function
11605  * @pf: pointer to i40e_pf struct
11606  **/
11607 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
11608 {
11609         struct i40e_hw *hw = &pf->hw;
11610         i40e_status ret;
11611         u8 mac_addr[6];
11612         u16 flags = 0;
11613
11614         /* Get current MAC address in case it's an LAA */
11615         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
11616                 ether_addr_copy(mac_addr,
11617                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
11618         } else {
11619                 dev_err(&pf->pdev->dev,
11620                         "Failed to retrieve MAC address; using default\n");
11621                 ether_addr_copy(mac_addr, hw->mac.addr);
11622         }
11623
11624         /* The FW expects the mac address write cmd to first be called with
11625          * one of these flags before calling it again with the multicast
11626          * enable flags.
11627          */
11628         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
11629
11630         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
11631                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
11632
11633         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11634         if (ret) {
11635                 dev_err(&pf->pdev->dev,
11636                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
11637                 return;
11638         }
11639
11640         flags = I40E_AQC_MC_MAG_EN
11641                         | I40E_AQC_WOL_PRESERVE_ON_PFR
11642                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
11643         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
11644         if (ret)
11645                 dev_err(&pf->pdev->dev,
11646                         "Failed to enable Multicast Magic Packet wake up\n");
11647 }
11648
11649 /**
11650  * i40e_shutdown - PCI callback for shutting down
11651  * @pdev: PCI device information struct
11652  **/
11653 static void i40e_shutdown(struct pci_dev *pdev)
11654 {
11655         struct i40e_pf *pf = pci_get_drvdata(pdev);
11656         struct i40e_hw *hw = &pf->hw;
11657
11658         set_bit(__I40E_SUSPENDED, &pf->state);
11659         set_bit(__I40E_DOWN, &pf->state);
11660         rtnl_lock();
11661         i40e_prep_for_reset(pf, true);
11662         rtnl_unlock();
11663
11664         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11665         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11666
11667         del_timer_sync(&pf->service_timer);
11668         cancel_work_sync(&pf->service_task);
11669         i40e_fdir_teardown(pf);
11670
11671         /* Client close must be called explicitly here because the timer
11672          * has been stopped.
11673          */
11674         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
11675
11676         if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
11677                 i40e_enable_mc_magic_wake(pf);
11678
11679         rtnl_lock();
11680         i40e_prep_for_reset(pf, true);
11681         rtnl_unlock();
11682
11683         wr32(hw, I40E_PFPM_APM,
11684              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11685         wr32(hw, I40E_PFPM_WUFC,
11686              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11687
11688         i40e_clear_interrupt_scheme(pf);
11689
11690         if (system_state == SYSTEM_POWER_OFF) {
11691                 pci_wake_from_d3(pdev, pf->wol_en);
11692                 pci_set_power_state(pdev, PCI_D3hot);
11693         }
11694 }
11695
11696 #ifdef CONFIG_PM
11697 /**
11698  * i40e_suspend - PCI callback for moving to D3
11699  * @pdev: PCI device information struct
11700  **/
11701 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
11702 {
11703         struct i40e_pf *pf = pci_get_drvdata(pdev);
11704         struct i40e_hw *hw = &pf->hw;
11705         int retval = 0;
11706
11707         set_bit(__I40E_SUSPENDED, &pf->state);
11708         set_bit(__I40E_DOWN, &pf->state);
11709
11710         if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE))
11711                 i40e_enable_mc_magic_wake(pf);
11712
11713         rtnl_lock();
11714         i40e_prep_for_reset(pf, true);
11715         rtnl_unlock();
11716
11717         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11718         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11719
11720         i40e_stop_misc_vector(pf);
11721
11722         retval = pci_save_state(pdev);
11723         if (retval)
11724                 return retval;
11725
11726         pci_wake_from_d3(pdev, pf->wol_en);
11727         pci_set_power_state(pdev, PCI_D3hot);
11728
11729         return retval;
11730 }
11731
11732 /**
11733  * i40e_resume - PCI callback for waking up from D3
11734  * @pdev: PCI device information struct
11735  **/
11736 static int i40e_resume(struct pci_dev *pdev)
11737 {
11738         struct i40e_pf *pf = pci_get_drvdata(pdev);
11739         u32 err;
11740
11741         pci_set_power_state(pdev, PCI_D0);
11742         pci_restore_state(pdev);
11743         /* pci_restore_state() clears dev->state_saves, so
11744          * call pci_save_state() again to restore it.
11745          */
11746         pci_save_state(pdev);
11747
11748         err = pci_enable_device_mem(pdev);
11749         if (err) {
11750                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
11751                 return err;
11752         }
11753         pci_set_master(pdev);
11754
11755         /* no wakeup events while running */
11756         pci_wake_from_d3(pdev, false);
11757
11758         /* handling the reset will rebuild the device state */
11759         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
11760                 clear_bit(__I40E_DOWN, &pf->state);
11761                 rtnl_lock();
11762                 i40e_reset_and_rebuild(pf, false, true);
11763                 rtnl_unlock();
11764         }
11765
11766         return 0;
11767 }
11768
11769 #endif
11770 static const struct pci_error_handlers i40e_err_handler = {
11771         .error_detected = i40e_pci_error_detected,
11772         .slot_reset = i40e_pci_error_slot_reset,
11773         .resume = i40e_pci_error_resume,
11774 };
11775
11776 static struct pci_driver i40e_driver = {
11777         .name     = i40e_driver_name,
11778         .id_table = i40e_pci_tbl,
11779         .probe    = i40e_probe,
11780         .remove   = i40e_remove,
11781 #ifdef CONFIG_PM
11782         .suspend  = i40e_suspend,
11783         .resume   = i40e_resume,
11784 #endif
11785         .shutdown = i40e_shutdown,
11786         .err_handler = &i40e_err_handler,
11787         .sriov_configure = i40e_pci_sriov_configure,
11788 };
11789
11790 /**
11791  * i40e_init_module - Driver registration routine
11792  *
11793  * i40e_init_module is the first routine called when the driver is
11794  * loaded. All it does is register with the PCI subsystem.
11795  **/
11796 static int __init i40e_init_module(void)
11797 {
11798         pr_info("%s: %s - version %s\n", i40e_driver_name,
11799                 i40e_driver_string, i40e_driver_version_str);
11800         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
11801
11802         /* we will see if single thread per module is enough for now,
11803          * it can't be any worse than using the system workqueue which
11804          * was already single threaded
11805          */
11806         i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
11807                                   i40e_driver_name);
11808         if (!i40e_wq) {
11809                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
11810                 return -ENOMEM;
11811         }
11812
11813         i40e_dbg_init();
11814         return pci_register_driver(&i40e_driver);
11815 }
11816 module_init(i40e_init_module);
11817
11818 /**
11819  * i40e_exit_module - Driver exit cleanup routine
11820  *
11821  * i40e_exit_module is called just before the driver is removed
11822  * from memory.
11823  **/
11824 static void __exit i40e_exit_module(void)
11825 {
11826         pci_unregister_driver(&i40e_driver);
11827         destroy_workqueue(i40e_wq);
11828         i40e_dbg_exit();
11829 }
11830 module_exit(i40e_exit_module);