drivers/net/ethernet/mellanox/mlx5/core/en_clock.c

   1 /*
   2  * Copyright (c) 2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/clocksource.h>
  34 #include "en.h"
  35
  36 enum {
  37         MLX5E_CYCLES_SHIFT      = 23
  38 };
  39
  40 enum {
  41         MLX5E_PIN_MODE_IN               = 0x0,
  42         MLX5E_PIN_MODE_OUT              = 0x1,
  43 };
  44
  45 enum {
  46         MLX5E_OUT_PATTERN_PULSE         = 0x0,
  47         MLX5E_OUT_PATTERN_PERIODIC      = 0x1,
  48 };
  49
  50 enum {
  51         MLX5E_EVENT_MODE_DISABLE        = 0x0,
  52         MLX5E_EVENT_MODE_REPETETIVE     = 0x1,
  53         MLX5E_EVENT_MODE_ONCE_TILL_ARM  = 0x2,
  54 };
  55
  56 void mlx5e_fill_hwstamp(struct mlx5e_tstamp *tstamp, u64 timestamp,
  57                         struct skb_shared_hwtstamps *hwts)
  58 {
  59         u64 nsec;
  60
  61         read_lock(&tstamp->lock);
  62         nsec = timecounter_cyc2time(&tstamp->clock, timestamp);
  63         read_unlock(&tstamp->lock);
  64
  65         hwts->hwtstamp = ns_to_ktime(nsec);
  66 }
  67
  68 static u64 mlx5e_read_internal_timer(const struct cyclecounter *cc)
  69 {
  70         struct mlx5e_tstamp *tstamp = container_of(cc, struct mlx5e_tstamp,
  71                                                    cycles);
  72
  73         return mlx5_read_internal_timer(tstamp->mdev) & cc->mask;
  74 }
  75
  76 static void mlx5e_timestamp_overflow(struct work_struct *work)
  77 {
  78         struct delayed_work *dwork = to_delayed_work(work);
  79         struct mlx5e_tstamp *tstamp = container_of(dwork, struct mlx5e_tstamp,
  80                                                    overflow_work);
  81         unsigned long flags;
  82
  83         write_lock_irqsave(&tstamp->lock, flags);
  84         timecounter_read(&tstamp->clock);
  85         write_unlock_irqrestore(&tstamp->lock, flags);
  86         schedule_delayed_work(&tstamp->overflow_work, tstamp->overflow_period);
  87 }
  88
  89 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
  90 {
  91         struct hwtstamp_config config;
  92         int err;
  93
  94         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
  95                 return -EOPNOTSUPP;
  96
  97         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
  98                 return -EFAULT;
  99
 100         /* TX HW timestamp */
 101         switch (config.tx_type) {
 102         case HWTSTAMP_TX_OFF:
 103         case HWTSTAMP_TX_ON:
 104                 break;
 105         default:
 106                 return -ERANGE;
 107         }
 108
 109         mutex_lock(&priv->state_lock);
 110         /* RX HW timestamp */
 111         switch (config.rx_filter) {
 112         case HWTSTAMP_FILTER_NONE:
 113                 /* Reset CQE compression to Admin default */
 114                 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def);
 115                 break;
 116         case HWTSTAMP_FILTER_ALL:
 117         case HWTSTAMP_FILTER_SOME:
 118         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 119         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 120         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 121         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 122         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 123         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 124         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 125         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 126         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 127         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 128         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 129         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 130         case HWTSTAMP_FILTER_NTP_ALL:
 131                 /* Disable CQE compression */
 132                 netdev_warn(priv->netdev, "Disabling cqe compression");
 133                 err = mlx5e_modify_rx_cqe_compression_locked(priv, false);
 134                 if (err) {
 135                         netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
 136                         mutex_unlock(&priv->state_lock);
 137                         return err;
 138                 }
 139                 config.rx_filter = HWTSTAMP_FILTER_ALL;
 140                 break;
 141         default:
 142                 mutex_unlock(&priv->state_lock);
 143                 return -ERANGE;
 144         }
 145
 146         memcpy(&priv->tstamp.hwtstamp_config, &config, sizeof(config));
 147         mutex_unlock(&priv->state_lock);
 148
 149         return copy_to_user(ifr->ifr_data, &config,
 150                             sizeof(config)) ? -EFAULT : 0;
 151 }
 152
 153 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
 154 {
 155         struct hwtstamp_config *cfg = &priv->tstamp.hwtstamp_config;
 156
 157         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
 158                 return -EOPNOTSUPP;
 159
 160         return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
 161 }
 162
 163 static int mlx5e_ptp_settime(struct ptp_clock_info *ptp,
 164                              const struct timespec64 *ts)
 165 {
 166         struct mlx5e_tstamp *tstamp = container_of(ptp, struct mlx5e_tstamp,
 167                                                    ptp_info);
 168         u64 ns = timespec64_to_ns(ts);
 169         unsigned long flags;
 170
 171         write_lock_irqsave(&tstamp->lock, flags);
 172         timecounter_init(&tstamp->clock, &tstamp->cycles, ns);
 173         write_unlock_irqrestore(&tstamp->lock, flags);
 174
 175         return 0;
 176 }
 177
 178 static int mlx5e_ptp_gettime(struct ptp_clock_info *ptp,
 179                              struct timespec64 *ts)
 180 {
 181         struct mlx5e_tstamp *tstamp = container_of(ptp, struct mlx5e_tstamp,
 182                                                    ptp_info);
 183         u64 ns;
 184         unsigned long flags;
 185
 186         write_lock_irqsave(&tstamp->lock, flags);
 187         ns = timecounter_read(&tstamp->clock);
 188         write_unlock_irqrestore(&tstamp->lock, flags);
 189
 190         *ts = ns_to_timespec64(ns);
 191
 192         return 0;
 193 }
 194
 195 static int mlx5e_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 196 {
 197         struct mlx5e_tstamp *tstamp = container_of(ptp, struct mlx5e_tstamp,
 198                                                    ptp_info);
 199         unsigned long flags;
 200
 201         write_lock_irqsave(&tstamp->lock, flags);
 202         timecounter_adjtime(&tstamp->clock, delta);
 203         write_unlock_irqrestore(&tstamp->lock, flags);
 204
 205         return 0;
 206 }
 207
 208 static int mlx5e_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta)
 209 {
 210         u64 adj;
 211         u32 diff;
 212         unsigned long flags;
 213         int neg_adj = 0;
 214         struct mlx5e_tstamp *tstamp = container_of(ptp, struct mlx5e_tstamp,
 215                                                   ptp_info);
 216         struct mlx5e_priv *priv =
 217                 container_of(tstamp, struct mlx5e_priv, tstamp);
 218
 219         if (MLX5_CAP_GEN(priv->mdev, pps_modify)) {
 220                 u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0};
 221
 222                 /* For future use need to add a loop for finding all 1PPS out pins */
 223                 MLX5_SET(mtpps_reg, in, pin_mode, MLX5E_PIN_MODE_OUT);
 224                 MLX5_SET(mtpps_reg, in, out_periodic_adjustment, delta & 0xFFFF);
 225
 226                 mlx5_set_mtpps(priv->mdev, in, sizeof(in));
 227         }
 228
 229         if (delta < 0) {
 230                 neg_adj = 1;
 231                 delta = -delta;
 232         }
 233
 234         adj = tstamp->nominal_c_mult;
 235         adj *= delta;
 236         diff = div_u64(adj, 1000000000ULL);
 237
 238         write_lock_irqsave(&tstamp->lock, flags);
 239         timecounter_read(&tstamp->clock);
 240         tstamp->cycles.mult = neg_adj ? tstamp->nominal_c_mult - diff :
 241                                         tstamp->nominal_c_mult + diff;
 242         write_unlock_irqrestore(&tstamp->lock, flags);
 243
 244         return 0;
 245 }
 246
 247 static int mlx5e_extts_configure(struct ptp_clock_info *ptp,
 248                                  struct ptp_clock_request *rq,
 249                                  int on)
 250 {
 251         struct mlx5e_tstamp *tstamp =
 252                 container_of(ptp, struct mlx5e_tstamp, ptp_info);
 253         struct mlx5e_priv *priv =
 254                 container_of(tstamp, struct mlx5e_priv, tstamp);
 255         u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0};
 256         u8 pattern = 0;
 257         int pin = -1;
 258         int err = 0;
 259
 260         if (!MLX5_CAP_GEN(priv->mdev, pps) ||
 261             !MLX5_CAP_GEN(priv->mdev, pps_modify))
 262                 return -EOPNOTSUPP;
 263
 264         if (rq->extts.index >= tstamp->ptp_info.n_pins)
 265                 return -EINVAL;
 266
 267         if (on) {
 268                 pin = ptp_find_pin(tstamp->ptp, PTP_PF_EXTTS, rq->extts.index);
 269                 if (pin < 0)
 270                         return -EBUSY;
 271         }
 272
 273         if (rq->extts.flags & PTP_FALLING_EDGE)
 274                 pattern = 1;
 275
 276         MLX5_SET(mtpps_reg, in, pin, pin);
 277         MLX5_SET(mtpps_reg, in, pin_mode, MLX5E_PIN_MODE_IN);
 278         MLX5_SET(mtpps_reg, in, pattern, pattern);
 279         MLX5_SET(mtpps_reg, in, enable, on);
 280
 281         err = mlx5_set_mtpps(priv->mdev, in, sizeof(in));
 282         if (err)
 283                 return err;
 284
 285         return mlx5_set_mtppse(priv->mdev, pin, 0,
 286                                MLX5E_EVENT_MODE_REPETETIVE & on);
 287 }
 288
 289 static int mlx5e_perout_configure(struct ptp_clock_info *ptp,
 290                                   struct ptp_clock_request *rq,
 291                                   int on)
 292 {
 293         struct mlx5e_tstamp *tstamp =
 294                 container_of(ptp, struct mlx5e_tstamp, ptp_info);
 295         struct mlx5e_priv *priv =
 296                 container_of(tstamp, struct mlx5e_priv, tstamp);
 297         u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0};
 298         u64 nsec_now, nsec_delta, time_stamp;
 299         u64 cycles_now, cycles_delta;
 300         struct timespec64 ts;
 301         unsigned long flags;
 302         int pin = -1;
 303         s64 ns;
 304
 305         if (!MLX5_CAP_GEN(priv->mdev, pps_modify))
 306                 return -EOPNOTSUPP;
 307
 308         if (rq->perout.index >= tstamp->ptp_info.n_pins)
 309                 return -EINVAL;
 310
 311         if (on) {
 312                 pin = ptp_find_pin(tstamp->ptp, PTP_PF_PEROUT,
 313                                    rq->perout.index);
 314                 if (pin < 0)
 315                         return -EBUSY;
 316         }
 317
 318         ts.tv_sec = rq->perout.period.sec;
 319         ts.tv_nsec = rq->perout.period.nsec;
 320         ns = timespec64_to_ns(&ts);
 321         if (on)
 322                 if ((ns >> 1) != 500000000LL)
 323                         return -EINVAL;
 324         ts.tv_sec = rq->perout.start.sec;
 325         ts.tv_nsec = rq->perout.start.nsec;
 326         ns = timespec64_to_ns(&ts);
 327         cycles_now = mlx5_read_internal_timer(tstamp->mdev);
 328         write_lock_irqsave(&tstamp->lock, flags);
 329         nsec_now = timecounter_cyc2time(&tstamp->clock, cycles_now);
 330         nsec_delta = ns - nsec_now;
 331         cycles_delta = div64_u64(nsec_delta << tstamp->cycles.shift,
 332                                  tstamp->cycles.mult);
 333         write_unlock_irqrestore(&tstamp->lock, flags);
 334         time_stamp = cycles_now + cycles_delta;
 335         MLX5_SET(mtpps_reg, in, pin, pin);
 336         MLX5_SET(mtpps_reg, in, pin_mode, MLX5E_PIN_MODE_OUT);
 337         MLX5_SET(mtpps_reg, in, pattern, MLX5E_OUT_PATTERN_PERIODIC);
 338         MLX5_SET(mtpps_reg, in, enable, on);
 339         MLX5_SET64(mtpps_reg, in, time_stamp, time_stamp);
 340
 341         return mlx5_set_mtpps(priv->mdev, in, sizeof(in));
 342 }
 343
 344 static int mlx5e_ptp_enable(struct ptp_clock_info *ptp,
 345                             struct ptp_clock_request *rq,
 346                             int on)
 347 {
 348         switch (rq->type) {
 349         case PTP_CLK_REQ_EXTTS:
 350                 return mlx5e_extts_configure(ptp, rq, on);
 351         case PTP_CLK_REQ_PEROUT:
 352                 return mlx5e_perout_configure(ptp, rq, on);
 353         default:
 354                 return -EOPNOTSUPP;
 355         }
 356         return 0;
 357 }
 358
 359 static int mlx5e_ptp_verify(struct ptp_clock_info *ptp, unsigned int pin,
 360                             enum ptp_pin_function func, unsigned int chan)
 361 {
 362         return (func == PTP_PF_PHYSYNC) ? -EOPNOTSUPP : 0;
 363 }
 364
 365 static const struct ptp_clock_info mlx5e_ptp_clock_info = {
 366         .owner          = THIS_MODULE,
 367         .max_adj        = 100000000,
 368         .n_alarm        = 0,
 369         .n_ext_ts       = 0,
 370         .n_per_out      = 0,
 371         .n_pins         = 0,
 372         .pps            = 0,
 373         .adjfreq        = mlx5e_ptp_adjfreq,
 374         .adjtime        = mlx5e_ptp_adjtime,
 375         .gettime64      = mlx5e_ptp_gettime,
 376         .settime64      = mlx5e_ptp_settime,
 377         .enable         = NULL,
 378         .verify         = NULL,
 379 };
 380
 381 static void mlx5e_timestamp_init_config(struct mlx5e_tstamp *tstamp)
 382 {
 383         tstamp->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF;
 384         tstamp->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
 385 }
 386
 387 static int mlx5e_init_pin_config(struct mlx5e_tstamp *tstamp)
 388 {
 389         int i;
 390
 391         tstamp->ptp_info.pin_config =
 392                 kzalloc(sizeof(*tstamp->ptp_info.pin_config) *
 393                                tstamp->ptp_info.n_pins, GFP_KERNEL);
 394         if (!tstamp->ptp_info.pin_config)
 395                 return -ENOMEM;
 396         tstamp->ptp_info.enable = mlx5e_ptp_enable;
 397         tstamp->ptp_info.verify = mlx5e_ptp_verify;
 398
 399         for (i = 0; i < tstamp->ptp_info.n_pins; i++) {
 400                 snprintf(tstamp->ptp_info.pin_config[i].name,
 401                          sizeof(tstamp->ptp_info.pin_config[i].name),
 402                          "mlx5_pps%d", i);
 403                 tstamp->ptp_info.pin_config[i].index = i;
 404                 tstamp->ptp_info.pin_config[i].func = PTP_PF_NONE;
 405                 tstamp->ptp_info.pin_config[i].chan = i;
 406         }
 407
 408         return 0;
 409 }
 410
 411 static void mlx5e_get_pps_caps(struct mlx5e_priv *priv,
 412                                struct mlx5e_tstamp *tstamp)
 413 {
 414         u32 out[MLX5_ST_SZ_DW(mtpps_reg)] = {0};
 415
 416         mlx5_query_mtpps(priv->mdev, out, sizeof(out));
 417
 418         tstamp->ptp_info.n_pins = MLX5_GET(mtpps_reg, out,
 419                                            cap_number_of_pps_pins);
 420         tstamp->ptp_info.n_ext_ts = MLX5_GET(mtpps_reg, out,
 421                                              cap_max_num_of_pps_in_pins);
 422         tstamp->ptp_info.n_per_out = MLX5_GET(mtpps_reg, out,
 423                                               cap_max_num_of_pps_out_pins);
 424
 425         tstamp->pps_pin_caps[0] = MLX5_GET(mtpps_reg, out, cap_pin_0_mode);
 426         tstamp->pps_pin_caps[1] = MLX5_GET(mtpps_reg, out, cap_pin_1_mode);
 427         tstamp->pps_pin_caps[2] = MLX5_GET(mtpps_reg, out, cap_pin_2_mode);
 428         tstamp->pps_pin_caps[3] = MLX5_GET(mtpps_reg, out, cap_pin_3_mode);
 429         tstamp->pps_pin_caps[4] = MLX5_GET(mtpps_reg, out, cap_pin_4_mode);
 430         tstamp->pps_pin_caps[5] = MLX5_GET(mtpps_reg, out, cap_pin_5_mode);
 431         tstamp->pps_pin_caps[6] = MLX5_GET(mtpps_reg, out, cap_pin_6_mode);
 432         tstamp->pps_pin_caps[7] = MLX5_GET(mtpps_reg, out, cap_pin_7_mode);
 433 }
 434
 435 void mlx5e_pps_event_handler(struct mlx5e_priv *priv,
 436                              struct ptp_clock_event *event)
 437 {
 438         struct mlx5e_tstamp *tstamp = &priv->tstamp;
 439
 440         ptp_clock_event(tstamp->ptp, event);
 441 }
 442
 443 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
 444 {
 445         struct mlx5e_tstamp *tstamp = &priv->tstamp;
 446         u64 ns;
 447         u64 frac = 0;
 448         u32 dev_freq;
 449
 450         mlx5e_timestamp_init_config(tstamp);
 451         dev_freq = MLX5_CAP_GEN(priv->mdev, device_frequency_khz);
 452         if (!dev_freq) {
 453                 mlx5_core_warn(priv->mdev, "invalid device_frequency_khz, aborting HW clock init\n");
 454                 return;
 455         }
 456         rwlock_init(&tstamp->lock);
 457         tstamp->cycles.read = mlx5e_read_internal_timer;
 458         tstamp->cycles.shift = MLX5E_CYCLES_SHIFT;
 459         tstamp->cycles.mult = clocksource_khz2mult(dev_freq,
 460                                                    tstamp->cycles.shift);
 461         tstamp->nominal_c_mult = tstamp->cycles.mult;
 462         tstamp->cycles.mask = CLOCKSOURCE_MASK(41);
 463         tstamp->mdev = priv->mdev;
 464
 465         timecounter_init(&tstamp->clock, &tstamp->cycles,
 466                          ktime_to_ns(ktime_get_real()));
 467
 468         /* Calculate period in seconds to call the overflow watchdog - to make
 469          * sure counter is checked at least once every wrap around.
 470          */
 471         ns = cyclecounter_cyc2ns(&tstamp->cycles, tstamp->cycles.mask,
 472                                  frac, &frac);
 473         do_div(ns, NSEC_PER_SEC / 2 / HZ);
 474         tstamp->overflow_period = ns;
 475
 476         INIT_DELAYED_WORK(&tstamp->overflow_work, mlx5e_timestamp_overflow);
 477         if (tstamp->overflow_period)
 478                 schedule_delayed_work(&tstamp->overflow_work, 0);
 479         else
 480                 mlx5_core_warn(priv->mdev, "invalid overflow period, overflow_work is not scheduled\n");
 481
 482         /* Configure the PHC */
 483         tstamp->ptp_info = mlx5e_ptp_clock_info;
 484         snprintf(tstamp->ptp_info.name, 16, "mlx5 ptp");
 485
 486         /* Initialize 1PPS data structures */
 487 #define MAX_PIN_NUM     8
 488         tstamp->pps_pin_caps = kzalloc(sizeof(u8) * MAX_PIN_NUM, GFP_KERNEL);
 489         if (tstamp->pps_pin_caps) {
 490                 if (MLX5_CAP_GEN(priv->mdev, pps))
 491                         mlx5e_get_pps_caps(priv, tstamp);
 492                 if (tstamp->ptp_info.n_pins)
 493                         mlx5e_init_pin_config(tstamp);
 494         } else {
 495                 mlx5_core_warn(priv->mdev, "1PPS initialization failed\n");
 496         }
 497
 498         tstamp->ptp = ptp_clock_register(&tstamp->ptp_info,
 499                                          &priv->mdev->pdev->dev);
 500         if (IS_ERR(tstamp->ptp)) {
 501                 mlx5_core_warn(priv->mdev, "ptp_clock_register failed %ld\n",
 502                                PTR_ERR(tstamp->ptp));
 503                 tstamp->ptp = NULL;
 504         }
 505 }
 506
 507 void mlx5e_timestamp_cleanup(struct mlx5e_priv *priv)
 508 {
 509         struct mlx5e_tstamp *tstamp = &priv->tstamp;
 510
 511         if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
 512                 return;
 513
 514         if (priv->tstamp.ptp) {
 515                 ptp_clock_unregister(priv->tstamp.ptp);
 516                 priv->tstamp.ptp = NULL;
 517         }
 518
 519         kfree(tstamp->pps_pin_caps);
 520         kfree(tstamp->ptp_info.pin_config);
 521
 522         cancel_delayed_work_sync(&tstamp->overflow_work);
 523 }