drivers/base/power/domain.c

   1 /*
   2  * drivers/base/power/domain.c - Common code related to device power domains.
   3  *
   4  * Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Renesas Electronics Corp.
   5  *
   6  * This file is released under the GPLv2.
   7  */
   8
   9 #include <linux/kernel.h>
  10 #include <linux/io.h>
  11 #include <linux/pm_runtime.h>
  12 #include <linux/pm_domain.h>
  13 #include <linux/pm_qos.h>
  14 #include <linux/slab.h>
  15 #include <linux/err.h>
  16 #include <linux/sched.h>
  17 #include <linux/suspend.h>
  18 #include <linux/export.h>
  19
  20 #define GENPD_DEV_CALLBACK(genpd, type, callback, dev)          \
  21 ({                                                              \
  22         type (*__routine)(struct device *__d);                  \
  23         type __ret = (type)0;                                   \
  24                                                                 \
  25         __routine = genpd->dev_ops.callback;                    \
  26         if (__routine) {                                        \
  27                 __ret = __routine(dev);                         \
  28         }                                                       \
  29         __ret;                                                  \
  30 })
  31
  32 #define GENPD_DEV_TIMED_CALLBACK(genpd, type, callback, dev, field, name)       \
  33 ({                                                                              \
  34         ktime_t __start = ktime_get();                                          \
  35         type __retval = GENPD_DEV_CALLBACK(genpd, type, callback, dev);         \
  36         s64 __elapsed = ktime_to_ns(ktime_sub(ktime_get(), __start));           \
  37         struct gpd_timing_data *__td = &dev_gpd_data(dev)->td;                  \
  38         if (!__retval && __elapsed > __td->field) {                             \
  39                 __td->field = __elapsed;                                        \
  40                 dev_dbg(dev, name " latency exceeded, new value %lld ns\n",     \
  41                         __elapsed);                                             \
  42                 genpd->max_off_time_changed = true;                             \
  43                 __td->constraint_changed = true;                                \
  44         }                                                                       \
  45         __retval;                                                               \
  46 })
  47
  48 static LIST_HEAD(gpd_list);
  49 static DEFINE_MUTEX(gpd_list_lock);
  50
  51 static struct generic_pm_domain *pm_genpd_lookup_name(const char *domain_name)
  52 {
  53         struct generic_pm_domain *genpd = NULL, *gpd;
  54
  55         if (IS_ERR_OR_NULL(domain_name))
  56                 return NULL;
  57
  58         mutex_lock(&gpd_list_lock);
  59         list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
  60                 if (!strcmp(gpd->name, domain_name)) {
  61                         genpd = gpd;
  62                         break;
  63                 }
  64         }
  65         mutex_unlock(&gpd_list_lock);
  66         return genpd;
  67 }
  68
  69 struct generic_pm_domain *dev_to_genpd(struct device *dev)
  70 {
  71         if (IS_ERR_OR_NULL(dev->pm_domain))
  72                 return ERR_PTR(-EINVAL);
  73
  74         return pd_to_genpd(dev->pm_domain);
  75 }
  76
  77 static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev)
  78 {
  79         return GENPD_DEV_TIMED_CALLBACK(genpd, int, stop, dev,
  80                                         stop_latency_ns, "stop");
  81 }
  82
  83 static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev)
  84 {
  85         return GENPD_DEV_TIMED_CALLBACK(genpd, int, start, dev,
  86                                         start_latency_ns, "start");
  87 }
  88
  89 static bool genpd_sd_counter_dec(struct generic_pm_domain *genpd)
  90 {
  91         bool ret = false;
  92
  93         if (!WARN_ON(atomic_read(&genpd->sd_count) == 0))
  94                 ret = !!atomic_dec_and_test(&genpd->sd_count);
  95
  96         return ret;
  97 }
  98
  99 static void genpd_sd_counter_inc(struct generic_pm_domain *genpd)
 100 {
 101         atomic_inc(&genpd->sd_count);
 102         smp_mb__after_atomic();
 103 }
 104
 105 static void genpd_acquire_lock(struct generic_pm_domain *genpd)
 106 {
 107         DEFINE_WAIT(wait);
 108
 109         mutex_lock(&genpd->lock);
 110         /*
 111          * Wait for the domain to transition into either the active,
 112          * or the power off state.
 113          */
 114         for (;;) {
 115                 prepare_to_wait(&genpd->status_wait_queue, &wait,
 116                                 TASK_UNINTERRUPTIBLE);
 117                 if (genpd->status == GPD_STATE_ACTIVE
 118                     || genpd->status == GPD_STATE_POWER_OFF)
 119                         break;
 120                 mutex_unlock(&genpd->lock);
 121
 122                 schedule();
 123
 124                 mutex_lock(&genpd->lock);
 125         }
 126         finish_wait(&genpd->status_wait_queue, &wait);
 127 }
 128
 129 static void genpd_release_lock(struct generic_pm_domain *genpd)
 130 {
 131         mutex_unlock(&genpd->lock);
 132 }
 133
 134 static void genpd_set_active(struct generic_pm_domain *genpd)
 135 {
 136         if (genpd->resume_count == 0)
 137                 genpd->status = GPD_STATE_ACTIVE;
 138 }
 139
 140 static void genpd_recalc_cpu_exit_latency(struct generic_pm_domain *genpd)
 141 {
 142         s64 usecs64;
 143
 144         if (!genpd->cpu_data)
 145                 return;
 146
 147         usecs64 = genpd->power_on_latency_ns;
 148         do_div(usecs64, NSEC_PER_USEC);
 149         usecs64 += genpd->cpu_data->saved_exit_latency;
 150         genpd->cpu_data->idle_state->exit_latency = usecs64;
 151 }
 152
 153 /**
 154  * __pm_genpd_poweron - Restore power to a given PM domain and its masters.
 155  * @genpd: PM domain to power up.
 156  *
 157  * Restore power to @genpd and all of its masters so that it is possible to
 158  * resume a device belonging to it.
 159  */
 160 static int __pm_genpd_poweron(struct generic_pm_domain *genpd)
 161         __releases(&genpd->lock) __acquires(&genpd->lock)
 162 {
 163         struct gpd_link *link;
 164         DEFINE_WAIT(wait);
 165         int ret = 0;
 166
 167         /* If the domain's master is being waited for, we have to wait too. */
 168         for (;;) {
 169                 prepare_to_wait(&genpd->status_wait_queue, &wait,
 170                                 TASK_UNINTERRUPTIBLE);
 171                 if (genpd->status != GPD_STATE_WAIT_MASTER)
 172                         break;
 173                 mutex_unlock(&genpd->lock);
 174
 175                 schedule();
 176
 177                 mutex_lock(&genpd->lock);
 178         }
 179         finish_wait(&genpd->status_wait_queue, &wait);
 180
 181         if (genpd->status == GPD_STATE_ACTIVE
 182             || (genpd->prepared_count > 0 && genpd->suspend_power_off))
 183                 return 0;
 184
 185         if (genpd->status != GPD_STATE_POWER_OFF) {
 186                 genpd_set_active(genpd);
 187                 return 0;
 188         }
 189
 190         if (genpd->cpu_data) {
 191                 cpuidle_pause_and_lock();
 192                 genpd->cpu_data->idle_state->disabled = true;
 193                 cpuidle_resume_and_unlock();
 194                 goto out;
 195         }
 196
 197         /*
 198          * The list is guaranteed not to change while the loop below is being
 199          * executed, unless one of the masters' .power_on() callbacks fiddles
 200          * with it.
 201          */
 202         list_for_each_entry(link, &genpd->slave_links, slave_node) {
 203                 genpd_sd_counter_inc(link->master);
 204                 genpd->status = GPD_STATE_WAIT_MASTER;
 205
 206                 mutex_unlock(&genpd->lock);
 207
 208                 ret = pm_genpd_poweron(link->master);
 209
 210                 mutex_lock(&genpd->lock);
 211
 212                 /*
 213                  * The "wait for parent" status is guaranteed not to change
 214                  * while the master is powering on.
 215                  */
 216                 genpd->status = GPD_STATE_POWER_OFF;
 217                 wake_up_all(&genpd->status_wait_queue);
 218                 if (ret) {
 219                         genpd_sd_counter_dec(link->master);
 220                         goto err;
 221                 }
 222         }
 223
 224         if (genpd->power_on) {
 225                 ktime_t time_start = ktime_get();
 226                 s64 elapsed_ns;
 227
 228                 ret = genpd->power_on(genpd);
 229                 if (ret)
 230                         goto err;
 231
 232                 elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
 233                 if (elapsed_ns > genpd->power_on_latency_ns) {
 234                         genpd->power_on_latency_ns = elapsed_ns;
 235                         genpd->max_off_time_changed = true;
 236                         genpd_recalc_cpu_exit_latency(genpd);
 237                         if (genpd->name)
 238                                 pr_warning("%s: Power-on latency exceeded, "
 239                                         "new value %lld ns\n", genpd->name,
 240                                         elapsed_ns);
 241                 }
 242         }
 243
 244  out:
 245         genpd_set_active(genpd);
 246
 247         return 0;
 248
 249  err:
 250         list_for_each_entry_continue_reverse(link, &genpd->slave_links, slave_node)
 251                 genpd_sd_counter_dec(link->master);
 252
 253         return ret;
 254 }
 255
 256 /**
 257  * pm_genpd_poweron - Restore power to a given PM domain and its masters.
 258  * @genpd: PM domain to power up.
 259  */
 260 int pm_genpd_poweron(struct generic_pm_domain *genpd)
 261 {
 262         int ret;
 263
 264         mutex_lock(&genpd->lock);
 265         ret = __pm_genpd_poweron(genpd);
 266         mutex_unlock(&genpd->lock);
 267         return ret;
 268 }
 269
 270 /**
 271  * pm_genpd_name_poweron - Restore power to a given PM domain and its masters.
 272  * @domain_name: Name of the PM domain to power up.
 273  */
 274 int pm_genpd_name_poweron(const char *domain_name)
 275 {
 276         struct generic_pm_domain *genpd;
 277
 278         genpd = pm_genpd_lookup_name(domain_name);
 279         return genpd ? pm_genpd_poweron(genpd) : -EINVAL;
 280 }
 281
 282 #ifdef CONFIG_PM_RUNTIME
 283
 284 static int genpd_start_dev_no_timing(struct generic_pm_domain *genpd,
 285                                      struct device *dev)
 286 {
 287         return GENPD_DEV_CALLBACK(genpd, int, start, dev);
 288 }
 289
 290 static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev)
 291 {
 292         return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev,
 293                                         save_state_latency_ns, "state save");
 294 }
 295
 296 static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev)
 297 {
 298         return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev,
 299                                         restore_state_latency_ns,
 300                                         "state restore");
 301 }
 302
 303 static int genpd_dev_pm_qos_notifier(struct notifier_block *nb,
 304                                      unsigned long val, void *ptr)
 305 {
 306         struct generic_pm_domain_data *gpd_data;
 307         struct device *dev;
 308
 309         gpd_data = container_of(nb, struct generic_pm_domain_data, nb);
 310
 311         mutex_lock(&gpd_data->lock);
 312         dev = gpd_data->base.dev;
 313         if (!dev) {
 314                 mutex_unlock(&gpd_data->lock);
 315                 return NOTIFY_DONE;
 316         }
 317         mutex_unlock(&gpd_data->lock);
 318
 319         for (;;) {
 320                 struct generic_pm_domain *genpd;
 321                 struct pm_domain_data *pdd;
 322
 323                 spin_lock_irq(&dev->power.lock);
 324
 325                 pdd = dev->power.subsys_data ?
 326                                 dev->power.subsys_data->domain_data : NULL;
 327                 if (pdd && pdd->dev) {
 328                         to_gpd_data(pdd)->td.constraint_changed = true;
 329                         genpd = dev_to_genpd(dev);
 330                 } else {
 331                         genpd = ERR_PTR(-ENODATA);
 332                 }
 333
 334                 spin_unlock_irq(&dev->power.lock);
 335
 336                 if (!IS_ERR(genpd)) {
 337                         mutex_lock(&genpd->lock);
 338                         genpd->max_off_time_changed = true;
 339                         mutex_unlock(&genpd->lock);
 340                 }
 341
 342                 dev = dev->parent;
 343                 if (!dev || dev->power.ignore_children)
 344                         break;
 345         }
 346
 347         return NOTIFY_DONE;
 348 }
 349
 350 /**
 351  * __pm_genpd_save_device - Save the pre-suspend state of a device.
 352  * @pdd: Domain data of the device to save the state of.
 353  * @genpd: PM domain the device belongs to.
 354  */
 355 static int __pm_genpd_save_device(struct pm_domain_data *pdd,
 356                                   struct generic_pm_domain *genpd)
 357         __releases(&genpd->lock) __acquires(&genpd->lock)
 358 {
 359         struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
 360         struct device *dev = pdd->dev;
 361         int ret = 0;
 362
 363         if (gpd_data->need_restore)
 364                 return 0;
 365
 366         mutex_unlock(&genpd->lock);
 367
 368         genpd_start_dev(genpd, dev);
 369         ret = genpd_save_dev(genpd, dev);
 370         genpd_stop_dev(genpd, dev);
 371
 372         mutex_lock(&genpd->lock);
 373
 374         if (!ret)
 375                 gpd_data->need_restore = true;
 376
 377         return ret;
 378 }
 379
 380 /**
 381  * __pm_genpd_restore_device - Restore the pre-suspend state of a device.
 382  * @pdd: Domain data of the device to restore the state of.
 383  * @genpd: PM domain the device belongs to.
 384  */
 385 static void __pm_genpd_restore_device(struct pm_domain_data *pdd,
 386                                       struct generic_pm_domain *genpd)
 387         __releases(&genpd->lock) __acquires(&genpd->lock)
 388 {
 389         struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd);
 390         struct device *dev = pdd->dev;
 391         bool need_restore = gpd_data->need_restore;
 392
 393         gpd_data->need_restore = false;
 394         mutex_unlock(&genpd->lock);
 395
 396         genpd_start_dev(genpd, dev);
 397         if (need_restore)
 398                 genpd_restore_dev(genpd, dev);
 399
 400         mutex_lock(&genpd->lock);
 401 }
 402
 403 /**
 404  * genpd_abort_poweroff - Check if a PM domain power off should be aborted.
 405  * @genpd: PM domain to check.
 406  *
 407  * Return true if a PM domain's status changed to GPD_STATE_ACTIVE during
 408  * a "power off" operation, which means that a "power on" has occured in the
 409  * meantime, or if its resume_count field is different from zero, which means
 410  * that one of its devices has been resumed in the meantime.
 411  */
 412 static bool genpd_abort_poweroff(struct generic_pm_domain *genpd)
 413 {
 414         return genpd->status == GPD_STATE_WAIT_MASTER
 415                 || genpd->status == GPD_STATE_ACTIVE || genpd->resume_count > 0;
 416 }
 417
 418 /**
 419  * genpd_queue_power_off_work - Queue up the execution of pm_genpd_poweroff().
 420  * @genpd: PM domait to power off.
 421  *
 422  * Queue up the execution of pm_genpd_poweroff() unless it's already been done
 423  * before.
 424  */
 425 static void genpd_queue_power_off_work(struct generic_pm_domain *genpd)
 426 {
 427         queue_work(pm_wq, &genpd->power_off_work);
 428 }
 429
 430 /**
 431  * pm_genpd_poweroff - Remove power from a given PM domain.
 432  * @genpd: PM domain to power down.
 433  *
 434  * If all of the @genpd's devices have been suspended and all of its subdomains
 435  * have been powered down, run the runtime suspend callbacks provided by all of
 436  * the @genpd's devices' drivers and remove power from @genpd.
 437  */
 438 static int pm_genpd_poweroff(struct generic_pm_domain *genpd)
 439         __releases(&genpd->lock) __acquires(&genpd->lock)
 440 {
 441         struct pm_domain_data *pdd;
 442         struct gpd_link *link;
 443         unsigned int not_suspended;
 444         int ret = 0;
 445
 446  start:
 447         /*
 448          * Do not try to power off the domain in the following situations:
 449          * (1) The domain is already in the "power off" state.
 450          * (2) The domain is waiting for its master to power up.
 451          * (3) One of the domain's devices is being resumed right now.
 452          * (4) System suspend is in progress.
 453          */
 454         if (genpd->status == GPD_STATE_POWER_OFF
 455             || genpd->status == GPD_STATE_WAIT_MASTER
 456             || genpd->resume_count > 0 || genpd->prepared_count > 0)
 457                 return 0;
 458
 459         if (atomic_read(&genpd->sd_count) > 0)
 460                 return -EBUSY;
 461
 462         not_suspended = 0;
 463         list_for_each_entry(pdd, &genpd->dev_list, list_node) {
 464                 enum pm_qos_flags_status stat;
 465
 466                 stat = dev_pm_qos_flags(pdd->dev,
 467                                         PM_QOS_FLAG_NO_POWER_OFF
 468                                                 | PM_QOS_FLAG_REMOTE_WAKEUP);
 469                 if (stat > PM_QOS_FLAGS_NONE)
 470                         return -EBUSY;
 471
 472                 if (pdd->dev->driver && (!pm_runtime_suspended(pdd->dev)
 473                     || pdd->dev->power.irq_safe))
 474                         not_suspended++;
 475         }
 476
 477         if (not_suspended > genpd->in_progress)
 478                 return -EBUSY;
 479
 480         if (genpd->poweroff_task) {
 481                 /*
 482                  * Another instance of pm_genpd_poweroff() is executing
 483                  * callbacks, so tell it to start over and return.
 484                  */
 485                 genpd->status = GPD_STATE_REPEAT;
 486                 return 0;
 487         }
 488
 489         if (genpd->gov && genpd->gov->power_down_ok) {
 490                 if (!genpd->gov->power_down_ok(&genpd->domain))
 491                         return -EAGAIN;
 492         }
 493
 494         genpd->status = GPD_STATE_BUSY;
 495         genpd->poweroff_task = current;
 496
 497         list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) {
 498                 ret = atomic_read(&genpd->sd_count) == 0 ?
 499                         __pm_genpd_save_device(pdd, genpd) : -EBUSY;
 500
 501                 if (genpd_abort_poweroff(genpd))
 502                         goto out;
 503
 504                 if (ret) {
 505                         genpd_set_active(genpd);
 506                         goto out;
 507                 }
 508
 509                 if (genpd->status == GPD_STATE_REPEAT) {
 510                         genpd->poweroff_task = NULL;
 511                         goto start;
 512                 }
 513         }
 514
 515         if (genpd->cpu_data) {
 516                 /*
 517                  * If cpu_data is set, cpuidle should turn the domain off when
 518                  * the CPU in it is idle.  In that case we don't decrement the
 519                  * subdomain counts of the master domains, so that power is not
 520                  * removed from the current domain prematurely as a result of
 521                  * cutting off the masters' power.
 522                  */
 523                 genpd->status = GPD_STATE_POWER_OFF;
 524                 cpuidle_pause_and_lock();
 525                 genpd->cpu_data->idle_state->disabled = false;
 526                 cpuidle_resume_and_unlock();
 527                 goto out;
 528         }
 529
 530         if (genpd->power_off) {
 531                 ktime_t time_start;
 532                 s64 elapsed_ns;
 533
 534                 if (atomic_read(&genpd->sd_count) > 0) {
 535                         ret = -EBUSY;
 536                         goto out;
 537                 }
 538
 539                 time_start = ktime_get();
 540
 541                 /*
 542                  * If sd_count > 0 at this point, one of the subdomains hasn't
 543                  * managed to call pm_genpd_poweron() for the master yet after
 544                  * incrementing it.  In that case pm_genpd_poweron() will wait
 545                  * for us to drop the lock, so we can call .power_off() and let
 546                  * the pm_genpd_poweron() restore power for us (this shouldn't
 547                  * happen very often).
 548                  */
 549                 ret = genpd->power_off(genpd);
 550                 if (ret == -EBUSY) {
 551                         genpd_set_active(genpd);
 552                         goto out;
 553                 }
 554
 555                 elapsed_ns = ktime_to_ns(ktime_sub(ktime_get(), time_start));
 556                 if (elapsed_ns > genpd->power_off_latency_ns) {
 557                         genpd->power_off_latency_ns = elapsed_ns;
 558                         genpd->max_off_time_changed = true;
 559                         if (genpd->name)
 560                                 pr_warning("%s: Power-off latency exceeded, "
 561                                         "new value %lld ns\n", genpd->name,
 562                                         elapsed_ns);
 563                 }
 564         }
 565
 566         genpd->status = GPD_STATE_POWER_OFF;
 567
 568         list_for_each_entry(link, &genpd->slave_links, slave_node) {
 569                 genpd_sd_counter_dec(link->master);
 570                 genpd_queue_power_off_work(link->master);
 571         }
 572
 573  out:
 574         genpd->poweroff_task = NULL;
 575         wake_up_all(&genpd->status_wait_queue);
 576         return ret;
 577 }
 578
 579 /**
 580  * genpd_power_off_work_fn - Power off PM domain whose subdomain count is 0.
 581  * @work: Work structure used for scheduling the execution of this function.
 582  */
 583 static void genpd_power_off_work_fn(struct work_struct *work)
 584 {
 585         struct generic_pm_domain *genpd;
 586
 587         genpd = container_of(work, struct generic_pm_domain, power_off_work);
 588
 589         genpd_acquire_lock(genpd);
 590         pm_genpd_poweroff(genpd);
 591         genpd_release_lock(genpd);
 592 }
 593
 594 /**
 595  * pm_genpd_runtime_suspend - Suspend a device belonging to I/O PM domain.
 596  * @dev: Device to suspend.
 597  *
 598  * Carry out a runtime suspend of a device under the assumption that its
 599  * pm_domain field points to the domain member of an object of type
 600  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 601  */
 602 static int pm_genpd_runtime_suspend(struct device *dev)
 603 {
 604         struct generic_pm_domain *genpd;
 605         bool (*stop_ok)(struct device *__dev);
 606         int ret;
 607
 608         dev_dbg(dev, "%s()\n", __func__);
 609
 610         genpd = dev_to_genpd(dev);
 611         if (IS_ERR(genpd))
 612                 return -EINVAL;
 613
 614         stop_ok = genpd->gov ? genpd->gov->stop_ok : NULL;
 615         if (stop_ok && !stop_ok(dev))
 616                 return -EBUSY;
 617
 618         ret = genpd_stop_dev(genpd, dev);
 619         if (ret)
 620                 return ret;
 621
 622         /*
 623          * If power.irq_safe is set, this routine will be run with interrupts
 624          * off, so it can't use mutexes.
 625          */
 626         if (dev->power.irq_safe)
 627                 return 0;
 628
 629         mutex_lock(&genpd->lock);
 630         genpd->in_progress++;
 631         pm_genpd_poweroff(genpd);
 632         genpd->in_progress--;
 633         mutex_unlock(&genpd->lock);
 634
 635         return 0;
 636 }
 637
 638 /**
 639  * pm_genpd_runtime_resume - Resume a device belonging to I/O PM domain.
 640  * @dev: Device to resume.
 641  *
 642  * Carry out a runtime resume of a device under the assumption that its
 643  * pm_domain field points to the domain member of an object of type
 644  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 645  */
 646 static int pm_genpd_runtime_resume(struct device *dev)
 647 {
 648         struct generic_pm_domain *genpd;
 649         DEFINE_WAIT(wait);
 650         int ret;
 651
 652         dev_dbg(dev, "%s()\n", __func__);
 653
 654         genpd = dev_to_genpd(dev);
 655         if (IS_ERR(genpd))
 656                 return -EINVAL;
 657
 658         /* If power.irq_safe, the PM domain is never powered off. */
 659         if (dev->power.irq_safe)
 660                 return genpd_start_dev_no_timing(genpd, dev);
 661
 662         mutex_lock(&genpd->lock);
 663         ret = __pm_genpd_poweron(genpd);
 664         if (ret) {
 665                 mutex_unlock(&genpd->lock);
 666                 return ret;
 667         }
 668         genpd->status = GPD_STATE_BUSY;
 669         genpd->resume_count++;
 670         for (;;) {
 671                 prepare_to_wait(&genpd->status_wait_queue, &wait,
 672                                 TASK_UNINTERRUPTIBLE);
 673                 /*
 674                  * If current is the powering off task, we have been called
 675                  * reentrantly from one of the device callbacks, so we should
 676                  * not wait.
 677                  */
 678                 if (!genpd->poweroff_task || genpd->poweroff_task == current)
 679                         break;
 680                 mutex_unlock(&genpd->lock);
 681
 682                 schedule();
 683
 684                 mutex_lock(&genpd->lock);
 685         }
 686         finish_wait(&genpd->status_wait_queue, &wait);
 687         __pm_genpd_restore_device(dev->power.subsys_data->domain_data, genpd);
 688         genpd->resume_count--;
 689         genpd_set_active(genpd);
 690         wake_up_all(&genpd->status_wait_queue);
 691         mutex_unlock(&genpd->lock);
 692
 693         return 0;
 694 }
 695
 696 static bool pd_ignore_unused;
 697 static int __init pd_ignore_unused_setup(char *__unused)
 698 {
 699         pd_ignore_unused = true;
 700         return 1;
 701 }
 702 __setup("pd_ignore_unused", pd_ignore_unused_setup);
 703
 704 /**
 705  * pm_genpd_poweroff_unused - Power off all PM domains with no devices in use.
 706  */
 707 void pm_genpd_poweroff_unused(void)
 708 {
 709         struct generic_pm_domain *genpd;
 710
 711         if (pd_ignore_unused) {
 712                 pr_warn("genpd: Not disabling unused power domains\n");
 713                 return;
 714         }
 715
 716         mutex_lock(&gpd_list_lock);
 717
 718         list_for_each_entry(genpd, &gpd_list, gpd_list_node)
 719                 genpd_queue_power_off_work(genpd);
 720
 721         mutex_unlock(&gpd_list_lock);
 722 }
 723
 724 #else
 725
 726 static inline int genpd_dev_pm_qos_notifier(struct notifier_block *nb,
 727                                             unsigned long val, void *ptr)
 728 {
 729         return NOTIFY_DONE;
 730 }
 731
 732 static inline void
 733 genpd_queue_power_off_work(struct generic_pm_domain *genpd) {}
 734
 735 static inline void genpd_power_off_work_fn(struct work_struct *work) {}
 736
 737 #define pm_genpd_runtime_suspend        NULL
 738 #define pm_genpd_runtime_resume         NULL
 739
 740 #endif /* CONFIG_PM_RUNTIME */
 741
 742 #ifdef CONFIG_PM_SLEEP
 743
 744 /**
 745  * pm_genpd_present - Check if the given PM domain has been initialized.
 746  * @genpd: PM domain to check.
 747  */
 748 static bool pm_genpd_present(struct generic_pm_domain *genpd)
 749 {
 750         struct generic_pm_domain *gpd;
 751
 752         if (IS_ERR_OR_NULL(genpd))
 753                 return false;
 754
 755         list_for_each_entry(gpd, &gpd_list, gpd_list_node)
 756                 if (gpd == genpd)
 757                         return true;
 758
 759         return false;
 760 }
 761
 762 static bool genpd_dev_active_wakeup(struct generic_pm_domain *genpd,
 763                                     struct device *dev)
 764 {
 765         return GENPD_DEV_CALLBACK(genpd, bool, active_wakeup, dev);
 766 }
 767
 768 /**
 769  * pm_genpd_sync_poweroff - Synchronously power off a PM domain and its masters.
 770  * @genpd: PM domain to power off, if possible.
 771  *
 772  * Check if the given PM domain can be powered off (during system suspend or
 773  * hibernation) and do that if so.  Also, in that case propagate to its masters.
 774  *
 775  * This function is only called in "noirq" and "syscore" stages of system power
 776  * transitions, so it need not acquire locks (all of the "noirq" callbacks are
 777  * executed sequentially, so it is guaranteed that it will never run twice in
 778  * parallel).
 779  */
 780 static void pm_genpd_sync_poweroff(struct generic_pm_domain *genpd)
 781 {
 782         struct gpd_link *link;
 783
 784         if (genpd->status == GPD_STATE_POWER_OFF)
 785                 return;
 786
 787         if (genpd->suspended_count != genpd->device_count
 788             || atomic_read(&genpd->sd_count) > 0)
 789                 return;
 790
 791         if (genpd->power_off)
 792                 genpd->power_off(genpd);
 793
 794         genpd->status = GPD_STATE_POWER_OFF;
 795
 796         list_for_each_entry(link, &genpd->slave_links, slave_node) {
 797                 genpd_sd_counter_dec(link->master);
 798                 pm_genpd_sync_poweroff(link->master);
 799         }
 800 }
 801
 802 /**
 803  * pm_genpd_sync_poweron - Synchronously power on a PM domain and its masters.
 804  * @genpd: PM domain to power on.
 805  *
 806  * This function is only called in "noirq" and "syscore" stages of system power
 807  * transitions, so it need not acquire locks (all of the "noirq" callbacks are
 808  * executed sequentially, so it is guaranteed that it will never run twice in
 809  * parallel).
 810  */
 811 static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd)
 812 {
 813         struct gpd_link *link;
 814
 815         if (genpd->status != GPD_STATE_POWER_OFF)
 816                 return;
 817
 818         list_for_each_entry(link, &genpd->slave_links, slave_node) {
 819                 pm_genpd_sync_poweron(link->master);
 820                 genpd_sd_counter_inc(link->master);
 821         }
 822
 823         if (genpd->power_on)
 824                 genpd->power_on(genpd);
 825
 826         genpd->status = GPD_STATE_ACTIVE;
 827 }
 828
 829 /**
 830  * resume_needed - Check whether to resume a device before system suspend.
 831  * @dev: Device to check.
 832  * @genpd: PM domain the device belongs to.
 833  *
 834  * There are two cases in which a device that can wake up the system from sleep
 835  * states should be resumed by pm_genpd_prepare(): (1) if the device is enabled
 836  * to wake up the system and it has to remain active for this purpose while the
 837  * system is in the sleep state and (2) if the device is not enabled to wake up
 838  * the system from sleep states and it generally doesn't generate wakeup signals
 839  * by itself (those signals are generated on its behalf by other parts of the
 840  * system).  In the latter case it may be necessary to reconfigure the device's
 841  * wakeup settings during system suspend, because it may have been set up to
 842  * signal remote wakeup from the system's working state as needed by runtime PM.
 843  * Return 'true' in either of the above cases.
 844  */
 845 static bool resume_needed(struct device *dev, struct generic_pm_domain *genpd)
 846 {
 847         bool active_wakeup;
 848
 849         if (!device_can_wakeup(dev))
 850                 return false;
 851
 852         active_wakeup = genpd_dev_active_wakeup(genpd, dev);
 853         return device_may_wakeup(dev) ? active_wakeup : !active_wakeup;
 854 }
 855
 856 /**
 857  * pm_genpd_prepare - Start power transition of a device in a PM domain.
 858  * @dev: Device to start the transition of.
 859  *
 860  * Start a power transition of a device (during a system-wide power transition)
 861  * under the assumption that its pm_domain field points to the domain member of
 862  * an object of type struct generic_pm_domain representing a PM domain
 863  * consisting of I/O devices.
 864  */
 865 static int pm_genpd_prepare(struct device *dev)
 866 {
 867         struct generic_pm_domain *genpd;
 868         int ret;
 869
 870         dev_dbg(dev, "%s()\n", __func__);
 871
 872         genpd = dev_to_genpd(dev);
 873         if (IS_ERR(genpd))
 874                 return -EINVAL;
 875
 876         /*
 877          * If a wakeup request is pending for the device, it should be woken up
 878          * at this point and a system wakeup event should be reported if it's
 879          * set up to wake up the system from sleep states.
 880          */
 881         pm_runtime_get_noresume(dev);
 882         if (pm_runtime_barrier(dev) && device_may_wakeup(dev))
 883                 pm_wakeup_event(dev, 0);
 884
 885         if (pm_wakeup_pending()) {
 886                 pm_runtime_put(dev);
 887                 return -EBUSY;
 888         }
 889
 890         if (resume_needed(dev, genpd))
 891                 pm_runtime_resume(dev);
 892
 893         genpd_acquire_lock(genpd);
 894
 895         if (genpd->prepared_count++ == 0) {
 896                 genpd->suspended_count = 0;
 897                 genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF;
 898         }
 899
 900         genpd_release_lock(genpd);
 901
 902         if (genpd->suspend_power_off) {
 903                 pm_runtime_put_noidle(dev);
 904                 return 0;
 905         }
 906
 907         /*
 908          * The PM domain must be in the GPD_STATE_ACTIVE state at this point,
 909          * so pm_genpd_poweron() will return immediately, but if the device
 910          * is suspended (e.g. it's been stopped by genpd_stop_dev()), we need
 911          * to make it operational.
 912          */
 913         pm_runtime_resume(dev);
 914         __pm_runtime_disable(dev, false);
 915
 916         ret = pm_generic_prepare(dev);
 917         if (ret) {
 918                 mutex_lock(&genpd->lock);
 919
 920                 if (--genpd->prepared_count == 0)
 921                         genpd->suspend_power_off = false;
 922
 923                 mutex_unlock(&genpd->lock);
 924                 pm_runtime_enable(dev);
 925         }
 926
 927         pm_runtime_put(dev);
 928         return ret;
 929 }
 930
 931 /**
 932  * pm_genpd_suspend - Suspend a device belonging to an I/O PM domain.
 933  * @dev: Device to suspend.
 934  *
 935  * Suspend a device under the assumption that its pm_domain field points to the
 936  * domain member of an object of type struct generic_pm_domain representing
 937  * a PM domain consisting of I/O devices.
 938  */
 939 static int pm_genpd_suspend(struct device *dev)
 940 {
 941         struct generic_pm_domain *genpd;
 942
 943         dev_dbg(dev, "%s()\n", __func__);
 944
 945         genpd = dev_to_genpd(dev);
 946         if (IS_ERR(genpd))
 947                 return -EINVAL;
 948
 949         return genpd->suspend_power_off ? 0 : pm_generic_suspend(dev);
 950 }
 951
 952 /**
 953  * pm_genpd_suspend_late - Late suspend of a device from an I/O PM domain.
 954  * @dev: Device to suspend.
 955  *
 956  * Carry out a late suspend of a device under the assumption that its
 957  * pm_domain field points to the domain member of an object of type
 958  * struct generic_pm_domain representing a PM domain consisting of I/O devices.
 959  */
 960 static int pm_genpd_suspend_late(struct device *dev)
 961 {
 962         struct generic_pm_domain *genpd;
 963
 964         dev_dbg(dev, "%s()\n", __func__);
 965
 966         genpd = dev_to_genpd(dev);
 967         if (IS_ERR(genpd))
 968                 return -EINVAL;
 969
 970         return genpd->suspend_power_off ? 0 : pm_generic_suspend_late(dev);
 971 }
 972
 973 /**
 974  * pm_genpd_suspend_noirq - Completion of suspend of device in an I/O PM domain.
 975  * @dev: Device to suspend.
 976  *
 977  * Stop the device and remove power from the domain if all devices in it have
 978  * been stopped.
 979  */
 980 static int pm_genpd_suspend_noirq(struct device *dev)
 981 {
 982         struct generic_pm_domain *genpd;
 983
 984         dev_dbg(dev, "%s()\n", __func__);
 985
 986         genpd = dev_to_genpd(dev);
 987         if (IS_ERR(genpd))
 988                 return -EINVAL;
 989
 990         if (genpd->suspend_power_off
 991             || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
 992                 return 0;
 993
 994         genpd_stop_dev(genpd, dev);
 995
 996         /*
 997          * Since all of the "noirq" callbacks are executed sequentially, it is
 998          * guaranteed that this function will never run twice in parallel for
 999          * the same PM domain, so it is not necessary to use locking here.
1000          */
1001         genpd->suspended_count++;
1002         pm_genpd_sync_poweroff(genpd);
1003
1004         return 0;
1005 }
1006
1007 /**
1008  * pm_genpd_resume_noirq - Start of resume of device in an I/O PM domain.
1009  * @dev: Device to resume.
1010  *
1011  * Restore power to the device's PM domain, if necessary, and start the device.
1012  */
1013 static int pm_genpd_resume_noirq(struct device *dev)
1014 {
1015         struct generic_pm_domain *genpd;
1016
1017         dev_dbg(dev, "%s()\n", __func__);
1018
1019         genpd = dev_to_genpd(dev);
1020         if (IS_ERR(genpd))
1021                 return -EINVAL;
1022
1023         if (genpd->suspend_power_off
1024             || (dev->power.wakeup_path && genpd_dev_active_wakeup(genpd, dev)))
1025                 return 0;
1026
1027         /*
1028          * Since all of the "noirq" callbacks are executed sequentially, it is
1029          * guaranteed that this function will never run twice in parallel for
1030          * the same PM domain, so it is not necessary to use locking here.
1031          */
1032         pm_genpd_sync_poweron(genpd);
1033         genpd->suspended_count--;
1034
1035         return genpd_start_dev(genpd, dev);
1036 }
1037
1038 /**
1039  * pm_genpd_resume_early - Early resume of a device in an I/O PM domain.
1040  * @dev: Device to resume.
1041  *
1042  * Carry out an early resume of a device under the assumption that its
1043  * pm_domain field points to the domain member of an object of type
1044  * struct generic_pm_domain representing a power domain consisting of I/O
1045  * devices.
1046  */
1047 static int pm_genpd_resume_early(struct device *dev)
1048 {
1049         struct generic_pm_domain *genpd;
1050
1051         dev_dbg(dev, "%s()\n", __func__);
1052
1053         genpd = dev_to_genpd(dev);
1054         if (IS_ERR(genpd))
1055                 return -EINVAL;
1056
1057         return genpd->suspend_power_off ? 0 : pm_generic_resume_early(dev);
1058 }
1059
1060 /**
1061  * pm_genpd_resume - Resume of device in an I/O PM domain.
1062  * @dev: Device to resume.
1063  *
1064  * Resume a device under the assumption that its pm_domain field points to the
1065  * domain member of an object of type struct generic_pm_domain representing
1066  * a power domain consisting of I/O devices.
1067  */
1068 static int pm_genpd_resume(struct device *dev)
1069 {
1070         struct generic_pm_domain *genpd;
1071
1072         dev_dbg(dev, "%s()\n", __func__);
1073
1074         genpd = dev_to_genpd(dev);
1075         if (IS_ERR(genpd))
1076                 return -EINVAL;
1077
1078         return genpd->suspend_power_off ? 0 : pm_generic_resume(dev);
1079 }
1080
1081 /**
1082  * pm_genpd_freeze - Freezing a device in an I/O PM domain.
1083  * @dev: Device to freeze.
1084  *
1085  * Freeze a device under the assumption that its pm_domain field points to the
1086  * domain member of an object of type struct generic_pm_domain representing
1087  * a power domain consisting of I/O devices.
1088  */
1089 static int pm_genpd_freeze(struct device *dev)
1090 {
1091         struct generic_pm_domain *genpd;
1092
1093         dev_dbg(dev, "%s()\n", __func__);
1094
1095         genpd = dev_to_genpd(dev);
1096         if (IS_ERR(genpd))
1097                 return -EINVAL;
1098
1099         return genpd->suspend_power_off ? 0 : pm_generic_freeze(dev);
1100 }
1101
1102 /**
1103  * pm_genpd_freeze_late - Late freeze of a device in an I/O PM domain.
1104  * @dev: Device to freeze.
1105  *
1106  * Carry out a late freeze of a device under the assumption that its
1107  * pm_domain field points to the domain member of an object of type
1108  * struct generic_pm_domain representing a power domain consisting of I/O
1109  * devices.
1110  */
1111 static int pm_genpd_freeze_late(struct device *dev)
1112 {
1113         struct generic_pm_domain *genpd;
1114
1115         dev_dbg(dev, "%s()\n", __func__);
1116
1117         genpd = dev_to_genpd(dev);
1118         if (IS_ERR(genpd))
1119                 return -EINVAL;
1120
1121         return genpd->suspend_power_off ? 0 : pm_generic_freeze_late(dev);
1122 }
1123
1124 /**
1125  * pm_genpd_freeze_noirq - Completion of freezing a device in an I/O PM domain.
1126  * @dev: Device to freeze.
1127  *
1128  * Carry out a late freeze of a device under the assumption that its
1129  * pm_domain field points to the domain member of an object of type
1130  * struct generic_pm_domain representing a power domain consisting of I/O
1131  * devices.
1132  */
1133 static int pm_genpd_freeze_noirq(struct device *dev)
1134 {
1135         struct generic_pm_domain *genpd;
1136
1137         dev_dbg(dev, "%s()\n", __func__);
1138
1139         genpd = dev_to_genpd(dev);
1140         if (IS_ERR(genpd))
1141                 return -EINVAL;
1142
1143         return genpd->suspend_power_off ? 0 : genpd_stop_dev(genpd, dev);
1144 }
1145
1146 /**
1147  * pm_genpd_thaw_noirq - Early thaw of device in an I/O PM domain.
1148  * @dev: Device to thaw.
1149  *
1150  * Start the device, unless power has been removed from the domain already
1151  * before the system transition.
1152  */
1153 static int pm_genpd_thaw_noirq(struct device *dev)
1154 {
1155         struct generic_pm_domain *genpd;
1156
1157         dev_dbg(dev, "%s()\n", __func__);
1158
1159         genpd = dev_to_genpd(dev);
1160         if (IS_ERR(genpd))
1161                 return -EINVAL;
1162
1163         return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev);
1164 }
1165
1166 /**
1167  * pm_genpd_thaw_early - Early thaw of device in an I/O PM domain.
1168  * @dev: Device to thaw.
1169  *
1170  * Carry out an early thaw of a device under the assumption that its
1171  * pm_domain field points to the domain member of an object of type
1172  * struct generic_pm_domain representing a power domain consisting of I/O
1173  * devices.
1174  */
1175 static int pm_genpd_thaw_early(struct device *dev)
1176 {
1177         struct generic_pm_domain *genpd;
1178
1179         dev_dbg(dev, "%s()\n", __func__);
1180
1181         genpd = dev_to_genpd(dev);
1182         if (IS_ERR(genpd))
1183                 return -EINVAL;
1184
1185         return genpd->suspend_power_off ? 0 : pm_generic_thaw_early(dev);
1186 }
1187
1188 /**
1189  * pm_genpd_thaw - Thaw a device belonging to an I/O power domain.
1190  * @dev: Device to thaw.
1191  *
1192  * Thaw a device under the assumption that its pm_domain field points to the
1193  * domain member of an object of type struct generic_pm_domain representing
1194  * a power domain consisting of I/O devices.
1195  */
1196 static int pm_genpd_thaw(struct device *dev)
1197 {
1198         struct generic_pm_domain *genpd;
1199
1200         dev_dbg(dev, "%s()\n", __func__);
1201
1202         genpd = dev_to_genpd(dev);
1203         if (IS_ERR(genpd))
1204                 return -EINVAL;
1205
1206         return genpd->suspend_power_off ? 0 : pm_generic_thaw(dev);
1207 }
1208
1209 /**
1210  * pm_genpd_restore_noirq - Start of restore of device in an I/O PM domain.
1211  * @dev: Device to resume.
1212  *
1213  * Make sure the domain will be in the same power state as before the
1214  * hibernation the system is resuming from and start the device if necessary.
1215  */
1216 static int pm_genpd_restore_noirq(struct device *dev)
1217 {
1218         struct generic_pm_domain *genpd;
1219
1220         dev_dbg(dev, "%s()\n", __func__);
1221
1222         genpd = dev_to_genpd(dev);
1223         if (IS_ERR(genpd))
1224                 return -EINVAL;
1225
1226         /*
1227          * Since all of the "noirq" callbacks are executed sequentially, it is
1228          * guaranteed that this function will never run twice in parallel for
1229          * the same PM domain, so it is not necessary to use locking here.
1230          *
1231          * At this point suspended_count == 0 means we are being run for the
1232          * first time for the given domain in the present cycle.
1233          */
1234         if (genpd->suspended_count++ == 0) {
1235                 /*
1236                  * The boot kernel might put the domain into arbitrary state,
1237                  * so make it appear as powered off to pm_genpd_sync_poweron(),
1238                  * so that it tries to power it on in case it was really off.
1239                  */
1240                 genpd->status = GPD_STATE_POWER_OFF;
1241                 if (genpd->suspend_power_off) {
1242                         /*
1243                          * If the domain was off before the hibernation, make
1244                          * sure it will be off going forward.
1245                          */
1246                         if (genpd->power_off)
1247                                 genpd->power_off(genpd);
1248
1249                         return 0;
1250                 }
1251         }
1252
1253         if (genpd->suspend_power_off)
1254                 return 0;
1255
1256         pm_genpd_sync_poweron(genpd);
1257
1258         return genpd_start_dev(genpd, dev);
1259 }
1260
1261 /**
1262  * pm_genpd_complete - Complete power transition of a device in a power domain.
1263  * @dev: Device to complete the transition of.
1264  *
1265  * Complete a power transition of a device (during a system-wide power
1266  * transition) under the assumption that its pm_domain field points to the
1267  * domain member of an object of type struct generic_pm_domain representing
1268  * a power domain consisting of I/O devices.
1269  */
1270 static void pm_genpd_complete(struct device *dev)
1271 {
1272         struct generic_pm_domain *genpd;
1273         bool run_complete;
1274
1275         dev_dbg(dev, "%s()\n", __func__);
1276
1277         genpd = dev_to_genpd(dev);
1278         if (IS_ERR(genpd))
1279                 return;
1280
1281         mutex_lock(&genpd->lock);
1282
1283         run_complete = !genpd->suspend_power_off;
1284         if (--genpd->prepared_count == 0)
1285                 genpd->suspend_power_off = false;
1286
1287         mutex_unlock(&genpd->lock);
1288
1289         if (run_complete) {
1290                 pm_generic_complete(dev);
1291                 pm_runtime_set_active(dev);
1292                 pm_runtime_enable(dev);
1293                 pm_request_idle(dev);
1294         }
1295 }
1296
1297 /**
1298  * genpd_syscore_switch - Switch power during system core suspend or resume.
1299  * @dev: Device that normally is marked as "always on" to switch power for.
1300  *
1301  * This routine may only be called during the system core (syscore) suspend or
1302  * resume phase for devices whose "always on" flags are set.
1303  */
1304 static void genpd_syscore_switch(struct device *dev, bool suspend)
1305 {
1306         struct generic_pm_domain *genpd;
1307
1308         genpd = dev_to_genpd(dev);
1309         if (!pm_genpd_present(genpd))
1310                 return;
1311
1312         if (suspend) {
1313                 genpd->suspended_count++;
1314                 pm_genpd_sync_poweroff(genpd);
1315         } else {
1316                 pm_genpd_sync_poweron(genpd);
1317                 genpd->suspended_count--;
1318         }
1319 }
1320
1321 void pm_genpd_syscore_poweroff(struct device *dev)
1322 {
1323         genpd_syscore_switch(dev, true);
1324 }
1325 EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweroff);
1326
1327 void pm_genpd_syscore_poweron(struct device *dev)
1328 {
1329         genpd_syscore_switch(dev, false);
1330 }
1331 EXPORT_SYMBOL_GPL(pm_genpd_syscore_poweron);
1332
1333 #else
1334
1335 #define pm_genpd_prepare                NULL
1336 #define pm_genpd_suspend                NULL
1337 #define pm_genpd_suspend_late           NULL
1338 #define pm_genpd_suspend_noirq          NULL
1339 #define pm_genpd_resume_early           NULL
1340 #define pm_genpd_resume_noirq           NULL
1341 #define pm_genpd_resume                 NULL
1342 #define pm_genpd_freeze                 NULL
1343 #define pm_genpd_freeze_late            NULL
1344 #define pm_genpd_freeze_noirq           NULL
1345 #define pm_genpd_thaw_early             NULL
1346 #define pm_genpd_thaw_noirq             NULL
1347 #define pm_genpd_thaw                   NULL
1348 #define pm_genpd_restore_noirq          NULL
1349 #define pm_genpd_complete               NULL
1350
1351 #endif /* CONFIG_PM_SLEEP */
1352
1353 static struct generic_pm_domain_data *__pm_genpd_alloc_dev_data(struct device *dev)
1354 {
1355         struct generic_pm_domain_data *gpd_data;
1356
1357         gpd_data = kzalloc(sizeof(*gpd_data), GFP_KERNEL);
1358         if (!gpd_data)
1359                 return NULL;
1360
1361         mutex_init(&gpd_data->lock);
1362         gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier;
1363         dev_pm_qos_add_notifier(dev, &gpd_data->nb);
1364         return gpd_data;
1365 }
1366
1367 static void __pm_genpd_free_dev_data(struct device *dev,
1368                                      struct generic_pm_domain_data *gpd_data)
1369 {
1370         dev_pm_qos_remove_notifier(dev, &gpd_data->nb);
1371         kfree(gpd_data);
1372 }
1373
1374 /**
1375  * __pm_genpd_add_device - Add a device to an I/O PM domain.
1376  * @genpd: PM domain to add the device to.
1377  * @dev: Device to be added.
1378  * @td: Set of PM QoS timing parameters to attach to the device.
1379  */
1380 int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev,
1381                           struct gpd_timing_data *td)
1382 {
1383         struct generic_pm_domain_data *gpd_data_new, *gpd_data = NULL;
1384         struct pm_domain_data *pdd;
1385         int ret = 0;
1386
1387         dev_dbg(dev, "%s()\n", __func__);
1388
1389         if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev))
1390                 return -EINVAL;
1391
1392         gpd_data_new = __pm_genpd_alloc_dev_data(dev);
1393         if (!gpd_data_new)
1394                 return -ENOMEM;
1395
1396         genpd_acquire_lock(genpd);
1397
1398         if (genpd->prepared_count > 0) {
1399                 ret = -EAGAIN;
1400                 goto out;
1401         }
1402
1403         list_for_each_entry(pdd, &genpd->dev_list, list_node)
1404                 if (pdd->dev == dev) {
1405                         ret = -EINVAL;
1406                         goto out;
1407                 }
1408
1409         ret = dev_pm_get_subsys_data(dev);
1410         if (ret)
1411                 goto out;
1412
1413         genpd->device_count++;
1414         genpd->max_off_time_changed = true;
1415
1416         spin_lock_irq(&dev->power.lock);
1417
1418         dev->pm_domain = &genpd->domain;
1419         if (dev->power.subsys_data->domain_data) {
1420                 gpd_data = to_gpd_data(dev->power.subsys_data->domain_data);
1421         } else {
1422                 gpd_data = gpd_data_new;
1423                 dev->power.subsys_data->domain_data = &gpd_data->base;
1424         }
1425         gpd_data->refcount++;
1426         if (td)
1427                 gpd_data->td = *td;
1428
1429         spin_unlock_irq(&dev->power.lock);
1430
1431         mutex_lock(&gpd_data->lock);
1432         gpd_data->base.dev = dev;
1433         list_add_tail(&gpd_data->base.list_node, &genpd->dev_list);
1434         gpd_data->need_restore = genpd->status == GPD_STATE_POWER_OFF;
1435         gpd_data->td.constraint_changed = true;
1436         gpd_data->td.effective_constraint_ns = -1;
1437         mutex_unlock(&gpd_data->lock);
1438
1439  out:
1440         genpd_release_lock(genpd);
1441
1442         if (gpd_data != gpd_data_new)
1443                 __pm_genpd_free_dev_data(dev, gpd_data_new);
1444
1445         return ret;
1446 }
1447
1448 /**
1449  * __pm_genpd_of_add_device - Add a device to an I/O PM domain.
1450  * @genpd_node: Device tree node pointer representing a PM domain to which the
1451  *   the device is added to.
1452  * @dev: Device to be added.
1453  * @td: Set of PM QoS timing parameters to attach to the device.
1454  */
1455 int __pm_genpd_of_add_device(struct device_node *genpd_node, struct device *dev,
1456                              struct gpd_timing_data *td)
1457 {
1458         struct generic_pm_domain *genpd = NULL, *gpd;
1459
1460         dev_dbg(dev, "%s()\n", __func__);
1461
1462         if (IS_ERR_OR_NULL(genpd_node) || IS_ERR_OR_NULL(dev))
1463                 return -EINVAL;
1464
1465         mutex_lock(&gpd_list_lock);
1466         list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
1467                 if (gpd->of_node == genpd_node) {
1468                         genpd = gpd;
1469                         break;
1470                 }
1471         }
1472         mutex_unlock(&gpd_list_lock);
1473
1474         if (!genpd)
1475                 return -EINVAL;
1476
1477         return __pm_genpd_add_device(genpd, dev, td);
1478 }
1479
1480
1481 /**
1482  * __pm_genpd_name_add_device - Find I/O PM domain and add a device to it.
1483  * @domain_name: Name of the PM domain to add the device to.
1484  * @dev: Device to be added.
1485  * @td: Set of PM QoS timing parameters to attach to the device.
1486  */
1487 int __pm_genpd_name_add_device(const char *domain_name, struct device *dev,
1488                                struct gpd_timing_data *td)
1489 {
1490         return __pm_genpd_add_device(pm_genpd_lookup_name(domain_name), dev, td);
1491 }
1492
1493 /**
1494  * pm_genpd_remove_device - Remove a device from an I/O PM domain.
1495  * @genpd: PM domain to remove the device from.
1496  * @dev: Device to be removed.
1497  */
1498 int pm_genpd_remove_device(struct generic_pm_domain *genpd,
1499                            struct device *dev)
1500 {
1501         struct generic_pm_domain_data *gpd_data;
1502         struct pm_domain_data *pdd;
1503         bool remove = false;
1504         int ret = 0;
1505
1506         dev_dbg(dev, "%s()\n", __func__);
1507
1508         if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(dev)
1509             ||  IS_ERR_OR_NULL(dev->pm_domain)
1510             ||  pd_to_genpd(dev->pm_domain) != genpd)
1511                 return -EINVAL;
1512
1513         genpd_acquire_lock(genpd);
1514
1515         if (genpd->prepared_count > 0) {
1516                 ret = -EAGAIN;
1517                 goto out;
1518         }
1519
1520         genpd->device_count--;
1521         genpd->max_off_time_changed = true;
1522
1523         spin_lock_irq(&dev->power.lock);
1524
1525         dev->pm_domain = NULL;
1526         pdd = dev->power.subsys_data->domain_data;
1527         list_del_init(&pdd->list_node);
1528         gpd_data = to_gpd_data(pdd);
1529         if (--gpd_data->refcount == 0) {
1530                 dev->power.subsys_data->domain_data = NULL;
1531                 remove = true;
1532         }
1533
1534         spin_unlock_irq(&dev->power.lock);
1535
1536         mutex_lock(&gpd_data->lock);
1537         pdd->dev = NULL;
1538         mutex_unlock(&gpd_data->lock);
1539
1540         genpd_release_lock(genpd);
1541
1542         dev_pm_put_subsys_data(dev);
1543         if (remove)
1544                 __pm_genpd_free_dev_data(dev, gpd_data);
1545
1546         return 0;
1547
1548  out:
1549         genpd_release_lock(genpd);
1550
1551         return ret;
1552 }
1553
1554 /**
1555  * pm_genpd_dev_need_restore - Set/unset the device's "need restore" flag.
1556  * @dev: Device to set/unset the flag for.
1557  * @val: The new value of the device's "need restore" flag.
1558  */
1559 void pm_genpd_dev_need_restore(struct device *dev, bool val)
1560 {
1561         struct pm_subsys_data *psd;
1562         unsigned long flags;
1563
1564         spin_lock_irqsave(&dev->power.lock, flags);
1565
1566         psd = dev_to_psd(dev);
1567         if (psd && psd->domain_data)
1568                 to_gpd_data(psd->domain_data)->need_restore = val;
1569
1570         spin_unlock_irqrestore(&dev->power.lock, flags);
1571 }
1572 EXPORT_SYMBOL_GPL(pm_genpd_dev_need_restore);
1573
1574 /**
1575  * pm_genpd_add_subdomain - Add a subdomain to an I/O PM domain.
1576  * @genpd: Master PM domain to add the subdomain to.
1577  * @subdomain: Subdomain to be added.
1578  */
1579 int pm_genpd_add_subdomain(struct generic_pm_domain *genpd,
1580                            struct generic_pm_domain *subdomain)
1581 {
1582         struct gpd_link *link;
1583         int ret = 0;
1584
1585         if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)
1586             || genpd == subdomain)
1587                 return -EINVAL;
1588
1589  start:
1590         genpd_acquire_lock(genpd);
1591         mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
1592
1593         if (subdomain->status != GPD_STATE_POWER_OFF
1594             && subdomain->status != GPD_STATE_ACTIVE) {
1595                 mutex_unlock(&subdomain->lock);
1596                 genpd_release_lock(genpd);
1597                 goto start;
1598         }
1599
1600         if (genpd->status == GPD_STATE_POWER_OFF
1601             &&  subdomain->status != GPD_STATE_POWER_OFF) {
1602                 ret = -EINVAL;
1603                 goto out;
1604         }
1605
1606         list_for_each_entry(link, &genpd->master_links, master_node) {
1607                 if (link->slave == subdomain && link->master == genpd) {
1608                         ret = -EINVAL;
1609                         goto out;
1610                 }
1611         }
1612
1613         link = kzalloc(sizeof(*link), GFP_KERNEL);
1614         if (!link) {
1615                 ret = -ENOMEM;
1616                 goto out;
1617         }
1618         link->master = genpd;
1619         list_add_tail(&link->master_node, &genpd->master_links);
1620         link->slave = subdomain;
1621         list_add_tail(&link->slave_node, &subdomain->slave_links);
1622         if (subdomain->status != GPD_STATE_POWER_OFF)
1623                 genpd_sd_counter_inc(genpd);
1624
1625  out:
1626         mutex_unlock(&subdomain->lock);
1627         genpd_release_lock(genpd);
1628
1629         return ret;
1630 }
1631
1632 /**
1633  * pm_genpd_add_subdomain_names - Add a subdomain to an I/O PM domain.
1634  * @master_name: Name of the master PM domain to add the subdomain to.
1635  * @subdomain_name: Name of the subdomain to be added.
1636  */
1637 int pm_genpd_add_subdomain_names(const char *master_name,
1638                                  const char *subdomain_name)
1639 {
1640         struct generic_pm_domain *master = NULL, *subdomain = NULL, *gpd;
1641
1642         if (IS_ERR_OR_NULL(master_name) || IS_ERR_OR_NULL(subdomain_name))
1643                 return -EINVAL;
1644
1645         mutex_lock(&gpd_list_lock);
1646         list_for_each_entry(gpd, &gpd_list, gpd_list_node) {
1647                 if (!master && !strcmp(gpd->name, master_name))
1648                         master = gpd;
1649
1650                 if (!subdomain && !strcmp(gpd->name, subdomain_name))
1651                         subdomain = gpd;
1652
1653                 if (master && subdomain)
1654                         break;
1655         }
1656         mutex_unlock(&gpd_list_lock);
1657
1658         return pm_genpd_add_subdomain(master, subdomain);
1659 }
1660
1661 /**
1662  * pm_genpd_remove_subdomain - Remove a subdomain from an I/O PM domain.
1663  * @genpd: Master PM domain to remove the subdomain from.
1664  * @subdomain: Subdomain to be removed.
1665  */
1666 int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd,
1667                               struct generic_pm_domain *subdomain)
1668 {
1669         struct gpd_link *link;
1670         int ret = -EINVAL;
1671
1672         if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain))
1673                 return -EINVAL;
1674
1675  start:
1676         genpd_acquire_lock(genpd);
1677
1678         list_for_each_entry(link, &genpd->master_links, master_node) {
1679                 if (link->slave != subdomain)
1680                         continue;
1681
1682                 mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING);
1683
1684                 if (subdomain->status != GPD_STATE_POWER_OFF
1685                     && subdomain->status != GPD_STATE_ACTIVE) {
1686                         mutex_unlock(&subdomain->lock);
1687                         genpd_release_lock(genpd);
1688                         goto start;
1689                 }
1690
1691                 list_del(&link->master_node);
1692                 list_del(&link->slave_node);
1693                 kfree(link);
1694                 if (subdomain->status != GPD_STATE_POWER_OFF)
1695                         genpd_sd_counter_dec(genpd);
1696
1697                 mutex_unlock(&subdomain->lock);
1698
1699                 ret = 0;
1700                 break;
1701         }
1702
1703         genpd_release_lock(genpd);
1704
1705         return ret;
1706 }
1707
1708 /**
1709  * pm_genpd_attach_cpuidle - Connect the given PM domain with cpuidle.
1710  * @genpd: PM domain to be connected with cpuidle.
1711  * @state: cpuidle state this domain can disable/enable.
1712  *
1713  * Make a PM domain behave as though it contained a CPU core, that is, instead
1714  * of calling its power down routine it will enable the given cpuidle state so
1715  * that the cpuidle subsystem can power it down (if possible and desirable).
1716  */
1717 int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state)
1718 {
1719         struct cpuidle_driver *cpuidle_drv;
1720         struct gpd_cpu_data *cpu_data;
1721         struct cpuidle_state *idle_state;
1722         int ret = 0;
1723
1724         if (IS_ERR_OR_NULL(genpd) || state < 0)
1725                 return -EINVAL;
1726
1727         genpd_acquire_lock(genpd);
1728
1729         if (genpd->cpu_data) {
1730                 ret = -EEXIST;
1731                 goto out;
1732         }
1733         cpu_data = kzalloc(sizeof(*cpu_data), GFP_KERNEL);
1734         if (!cpu_data) {
1735                 ret = -ENOMEM;
1736                 goto out;
1737         }
1738         cpuidle_drv = cpuidle_driver_ref();
1739         if (!cpuidle_drv) {
1740                 ret = -ENODEV;
1741                 goto err_drv;
1742         }
1743         if (cpuidle_drv->state_count <= state) {
1744                 ret = -EINVAL;
1745                 goto err;
1746         }
1747         idle_state = &cpuidle_drv->states[state];
1748         if (!idle_state->disabled) {
1749                 ret = -EAGAIN;
1750                 goto err;
1751         }
1752         cpu_data->idle_state = idle_state;
1753         cpu_data->saved_exit_latency = idle_state->exit_latency;
1754         genpd->cpu_data = cpu_data;
1755         genpd_recalc_cpu_exit_latency(genpd);
1756
1757  out:
1758         genpd_release_lock(genpd);
1759         return ret;
1760
1761  err:
1762         cpuidle_driver_unref();
1763
1764  err_drv:
1765         kfree(cpu_data);
1766         goto out;
1767 }
1768
1769 /**
1770  * pm_genpd_name_attach_cpuidle - Find PM domain and connect cpuidle to it.
1771  * @name: Name of the domain to connect to cpuidle.
1772  * @state: cpuidle state this domain can manipulate.
1773  */
1774 int pm_genpd_name_attach_cpuidle(const char *name, int state)
1775 {
1776         return pm_genpd_attach_cpuidle(pm_genpd_lookup_name(name), state);
1777 }
1778
1779 /**
1780  * pm_genpd_detach_cpuidle - Remove the cpuidle connection from a PM domain.
1781  * @genpd: PM domain to remove the cpuidle connection from.
1782  *
1783  * Remove the cpuidle connection set up by pm_genpd_attach_cpuidle() from the
1784  * given PM domain.
1785  */
1786 int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd)
1787 {
1788         struct gpd_cpu_data *cpu_data;
1789         struct cpuidle_state *idle_state;
1790         int ret = 0;
1791
1792         if (IS_ERR_OR_NULL(genpd))
1793                 return -EINVAL;
1794
1795         genpd_acquire_lock(genpd);
1796
1797         cpu_data = genpd->cpu_data;
1798         if (!cpu_data) {
1799                 ret = -ENODEV;
1800                 goto out;
1801         }
1802         idle_state = cpu_data->idle_state;
1803         if (!idle_state->disabled) {
1804                 ret = -EAGAIN;
1805                 goto out;
1806         }
1807         idle_state->exit_latency = cpu_data->saved_exit_latency;
1808         cpuidle_driver_unref();
1809         genpd->cpu_data = NULL;
1810         kfree(cpu_data);
1811
1812  out:
1813         genpd_release_lock(genpd);
1814         return ret;
1815 }
1816
1817 /**
1818  * pm_genpd_name_detach_cpuidle - Find PM domain and disconnect cpuidle from it.
1819  * @name: Name of the domain to disconnect cpuidle from.
1820  */
1821 int pm_genpd_name_detach_cpuidle(const char *name)
1822 {
1823         return pm_genpd_detach_cpuidle(pm_genpd_lookup_name(name));
1824 }
1825
1826 /* Default device callbacks for generic PM domains. */
1827
1828 /**
1829  * pm_genpd_default_save_state - Default "save device state" for PM domians.
1830  * @dev: Device to handle.
1831  */
1832 static int pm_genpd_default_save_state(struct device *dev)
1833 {
1834         int (*cb)(struct device *__dev);
1835
1836         if (dev->type && dev->type->pm)
1837                 cb = dev->type->pm->runtime_suspend;
1838         else if (dev->class && dev->class->pm)
1839                 cb = dev->class->pm->runtime_suspend;
1840         else if (dev->bus && dev->bus->pm)
1841                 cb = dev->bus->pm->runtime_suspend;
1842         else
1843                 cb = NULL;
1844
1845         if (!cb && dev->driver && dev->driver->pm)
1846                 cb = dev->driver->pm->runtime_suspend;
1847
1848         return cb ? cb(dev) : 0;
1849 }
1850
1851 /**
1852  * pm_genpd_default_restore_state - Default PM domians "restore device state".
1853  * @dev: Device to handle.
1854  */
1855 static int pm_genpd_default_restore_state(struct device *dev)
1856 {
1857         int (*cb)(struct device *__dev);
1858
1859         if (dev->type && dev->type->pm)
1860                 cb = dev->type->pm->runtime_resume;
1861         else if (dev->class && dev->class->pm)
1862                 cb = dev->class->pm->runtime_resume;
1863         else if (dev->bus && dev->bus->pm)
1864                 cb = dev->bus->pm->runtime_resume;
1865         else
1866                 cb = NULL;
1867
1868         if (!cb && dev->driver && dev->driver->pm)
1869                 cb = dev->driver->pm->runtime_resume;
1870
1871         return cb ? cb(dev) : 0;
1872 }
1873
1874 /**
1875  * pm_genpd_init - Initialize a generic I/O PM domain object.
1876  * @genpd: PM domain object to initialize.
1877  * @gov: PM domain governor to associate with the domain (may be NULL).
1878  * @is_off: Initial value of the domain's power_is_off field.
1879  */
1880 void pm_genpd_init(struct generic_pm_domain *genpd,
1881                    struct dev_power_governor *gov, bool is_off)
1882 {
1883         if (IS_ERR_OR_NULL(genpd))
1884                 return;
1885
1886         INIT_LIST_HEAD(&genpd->master_links);
1887         INIT_LIST_HEAD(&genpd->slave_links);
1888         INIT_LIST_HEAD(&genpd->dev_list);
1889         mutex_init(&genpd->lock);
1890         genpd->gov = gov;
1891         INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn);
1892         genpd->in_progress = 0;
1893         atomic_set(&genpd->sd_count, 0);
1894         genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE;
1895         init_waitqueue_head(&genpd->status_wait_queue);
1896         genpd->poweroff_task = NULL;
1897         genpd->resume_count = 0;
1898         genpd->device_count = 0;
1899         genpd->max_off_time_ns = -1;
1900         genpd->max_off_time_changed = true;
1901         genpd->domain.ops.runtime_suspend = pm_genpd_runtime_suspend;
1902         genpd->domain.ops.runtime_resume = pm_genpd_runtime_resume;
1903         genpd->domain.ops.prepare = pm_genpd_prepare;
1904         genpd->domain.ops.suspend = pm_genpd_suspend;
1905         genpd->domain.ops.suspend_late = pm_genpd_suspend_late;
1906         genpd->domain.ops.suspend_noirq = pm_genpd_suspend_noirq;
1907         genpd->domain.ops.resume_noirq = pm_genpd_resume_noirq;
1908         genpd->domain.ops.resume_early = pm_genpd_resume_early;
1909         genpd->domain.ops.resume = pm_genpd_resume;
1910         genpd->domain.ops.freeze = pm_genpd_freeze;
1911         genpd->domain.ops.freeze_late = pm_genpd_freeze_late;
1912         genpd->domain.ops.freeze_noirq = pm_genpd_freeze_noirq;
1913         genpd->domain.ops.thaw_noirq = pm_genpd_thaw_noirq;
1914         genpd->domain.ops.thaw_early = pm_genpd_thaw_early;
1915         genpd->domain.ops.thaw = pm_genpd_thaw;
1916         genpd->domain.ops.poweroff = pm_genpd_suspend;
1917         genpd->domain.ops.poweroff_late = pm_genpd_suspend_late;
1918         genpd->domain.ops.poweroff_noirq = pm_genpd_suspend_noirq;
1919         genpd->domain.ops.restore_noirq = pm_genpd_restore_noirq;
1920         genpd->domain.ops.restore_early = pm_genpd_resume_early;
1921         genpd->domain.ops.restore = pm_genpd_resume;
1922         genpd->domain.ops.complete = pm_genpd_complete;
1923         genpd->dev_ops.save_state = pm_genpd_default_save_state;
1924         genpd->dev_ops.restore_state = pm_genpd_default_restore_state;
1925         mutex_lock(&gpd_list_lock);
1926         list_add(&genpd->gpd_list_node, &gpd_list);
1927         mutex_unlock(&gpd_list_lock);
1928 }