2 * Generic OPP Interface
4 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/list.h>
21 #include <linux/rculist.h>
22 #include <linux/rcupdate.h>
23 #include <linux/pm_opp.h>
25 #include <linux/export.h>
28 * Internal data structure organization with the OPP layer library is as
31 * |- device 1 (represents voltage domain 1)
32 * | |- opp 1 (availability, freq, voltage)
36 * |- device 2 (represents the next voltage domain)
38 * `- device m (represents mth voltage domain)
39 * device 1, 2.. are represented by dev_opp structure while each opp
40 * is represented by the opp structure.
44 * struct dev_pm_opp - Generic OPP description structure
45 * @node: opp list node. The nodes are maintained throughout the lifetime
46 * of boot. It is expected only an optimal set of OPPs are
47 * added to the library by the SoC framework.
48 * RCU usage: opp list is traversed with RCU locks. node
49 * modification is possible realtime, hence the modifications
50 * are protected by the dev_opp_list_lock for integrity.
51 * IMPORTANT: the opp nodes should be maintained in increasing
53 * @dynamic: not-created from static DT entries.
54 * @available: true/false - marks if this OPP as available or not
55 * @turbo: true if turbo (boost) OPP
56 * @rate: Frequency in hertz
57 * @u_volt: Target voltage in microvolts corresponding to this OPP
58 * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP
59 * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP
60 * @u_amp: Maximum current drawn by the device in microamperes
61 * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
62 * frequency from any other OPP's frequency.
63 * @dev_opp: points back to the device_opp struct this opp belongs to
64 * @rcu_head: RCU callback head used for deferred freeing
65 * @np: OPP's device node.
67 * This structure stores the OPP information for a given device.
70 struct list_head node;
78 unsigned long u_volt_min;
79 unsigned long u_volt_max;
81 unsigned long clock_latency_ns;
83 struct device_opp *dev_opp;
84 struct rcu_head rcu_head;
86 struct device_node *np;
90 * struct device_list_opp - devices managed by 'struct device_opp'
92 * @dev: device to which the struct object belongs
93 * @rcu_head: RCU callback head used for deferred freeing
95 * This is an internal data structure maintaining the list of devices that are
96 * managed by 'struct device_opp'.
98 struct device_list_opp {
99 struct list_head node;
100 const struct device *dev;
101 struct rcu_head rcu_head;
105 * struct device_opp - Device opp structure
106 * @node: list node - contains the devices with OPPs that
107 * have been registered. Nodes once added are not modified in this
109 * RCU usage: nodes are not modified in the list of device_opp,
110 * however addition is possible and is secured by dev_opp_list_lock
111 * @srcu_head: notifier head to notify the OPP availability changes.
112 * @rcu_head: RCU callback head used for deferred freeing
113 * @dev_list: list of devices that share these OPPs
114 * @opp_list: list of opps
115 * @np: struct device_node pointer for opp's DT node.
116 * @shared_opp: OPP is shared between multiple devices.
118 * This is an internal data structure maintaining the link to opps attached to
119 * a device. This structure is not meant to be shared to users as it is
120 * meant for book keeping and private to OPP library.
122 * Because the opp structures can be used from both rcu and srcu readers, we
123 * need to wait for the grace period of both of them before freeing any
124 * resources. And so we have used kfree_rcu() from within call_srcu() handlers.
127 struct list_head node;
129 struct srcu_notifier_head srcu_head;
130 struct rcu_head rcu_head;
131 struct list_head dev_list;
132 struct list_head opp_list;
134 struct device_node *np;
135 unsigned long clock_latency_ns_max;
137 struct dev_pm_opp *suspend_opp;
141 * The root of the list of all devices. All device_opp structures branch off
142 * from here, with each device_opp containing the list of opp it supports in
143 * various states of availability.
145 static LIST_HEAD(dev_opp_list);
146 /* Lock to allow exclusive modification to the device and opp lists */
147 static DEFINE_MUTEX(dev_opp_list_lock);
149 #define opp_rcu_lockdep_assert() \
151 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
152 !lockdep_is_held(&dev_opp_list_lock), \
153 "Missing rcu_read_lock() or " \
154 "dev_opp_list_lock protection"); \
157 static struct device_list_opp *_find_list_dev(const struct device *dev,
158 struct device_opp *dev_opp)
160 struct device_list_opp *list_dev;
162 list_for_each_entry(list_dev, &dev_opp->dev_list, node)
163 if (list_dev->dev == dev)
169 static struct device_opp *_managed_opp(const struct device_node *np)
171 struct device_opp *dev_opp;
173 list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) {
174 if (dev_opp->np == np) {
176 * Multiple devices can point to the same OPP table and
177 * so will have same node-pointer, np.
179 * But the OPPs will be considered as shared only if the
180 * OPP table contains a "opp-shared" property.
182 return dev_opp->shared_opp ? dev_opp : NULL;
190 * _find_device_opp() - find device_opp struct using device pointer
191 * @dev: device pointer used to lookup device OPPs
193 * Search list of device OPPs for one containing matching device. Does a RCU
194 * reader operation to grab the pointer needed.
196 * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or
197 * -EINVAL based on type of error.
199 * Locking: This function must be called under rcu_read_lock(). device_opp
200 * is a RCU protected pointer. This means that device_opp is valid as long
201 * as we are under RCU lock.
203 static struct device_opp *_find_device_opp(struct device *dev)
205 struct device_opp *dev_opp;
207 if (IS_ERR_OR_NULL(dev)) {
208 pr_err("%s: Invalid parameters\n", __func__);
209 return ERR_PTR(-EINVAL);
212 list_for_each_entry_rcu(dev_opp, &dev_opp_list, node)
213 if (_find_list_dev(dev, dev_opp))
216 return ERR_PTR(-ENODEV);
220 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp
221 * @opp: opp for which voltage has to be returned for
223 * Return: voltage in micro volt corresponding to the opp, else
226 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
227 * protected pointer. This means that opp which could have been fetched by
228 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
229 * under RCU lock. The pointer returned by the opp_find_freq family must be
230 * used in the same section as the usage of this function with the pointer
231 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
234 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp)
236 struct dev_pm_opp *tmp_opp;
239 opp_rcu_lockdep_assert();
241 tmp_opp = rcu_dereference(opp);
242 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available)
243 pr_err("%s: Invalid parameters\n", __func__);
249 EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage);
252 * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp
253 * @opp: opp for which frequency has to be returned for
255 * Return: frequency in hertz corresponding to the opp, else
258 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
259 * protected pointer. This means that opp which could have been fetched by
260 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
261 * under RCU lock. The pointer returned by the opp_find_freq family must be
262 * used in the same section as the usage of this function with the pointer
263 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
266 unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp)
268 struct dev_pm_opp *tmp_opp;
271 opp_rcu_lockdep_assert();
273 tmp_opp = rcu_dereference(opp);
274 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available)
275 pr_err("%s: Invalid parameters\n", __func__);
281 EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq);
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
291 * Return: true if opp is turbo opp, else false.
293 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
294 * protected pointer. This means that opp which could have been fetched by
295 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
296 * under RCU lock. The pointer returned by the opp_find_freq family must be
297 * used in the same section as the usage of this function with the pointer
298 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
301 bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp)
303 struct dev_pm_opp *tmp_opp;
305 opp_rcu_lockdep_assert();
307 tmp_opp = rcu_dereference(opp);
308 if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) {
309 pr_err("%s: Invalid parameters\n", __func__);
313 return tmp_opp->turbo;
315 EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo);
318 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
319 * @dev: device for which we do this operation
321 * Return: This function returns the max clock latency in nanoseconds.
323 * Locking: This function takes rcu_read_lock().
325 unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev)
327 struct device_opp *dev_opp;
328 unsigned long clock_latency_ns;
332 dev_opp = _find_device_opp(dev);
334 clock_latency_ns = 0;
336 clock_latency_ns = dev_opp->clock_latency_ns_max;
339 return clock_latency_ns;
341 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency);
344 * dev_pm_opp_get_suspend_opp() - Get suspend opp
345 * @dev: device for which we do this operation
347 * Return: This function returns pointer to the suspend opp if it is
348 * defined and available, otherwise it returns NULL.
350 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
351 * protected pointer. The reason for the same is that the opp pointer which is
352 * returned will remain valid for use with opp_get_{voltage, freq} only while
353 * under the locked area. The pointer returned must be used prior to unlocking
354 * with rcu_read_unlock() to maintain the integrity of the pointer.
356 struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev)
358 struct device_opp *dev_opp;
360 opp_rcu_lockdep_assert();
362 dev_opp = _find_device_opp(dev);
363 if (IS_ERR(dev_opp) || !dev_opp->suspend_opp ||
364 !dev_opp->suspend_opp->available)
367 return dev_opp->suspend_opp;
369 EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp);
372 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list
373 * @dev: device for which we do this operation
375 * Return: This function returns the number of available opps if there are any,
376 * else returns 0 if none or the corresponding error value.
378 * Locking: This function takes rcu_read_lock().
380 int dev_pm_opp_get_opp_count(struct device *dev)
382 struct device_opp *dev_opp;
383 struct dev_pm_opp *temp_opp;
388 dev_opp = _find_device_opp(dev);
389 if (IS_ERR(dev_opp)) {
390 count = PTR_ERR(dev_opp);
391 dev_err(dev, "%s: device OPP not found (%d)\n",
396 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
397 if (temp_opp->available)
405 EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count);
408 * dev_pm_opp_find_freq_exact() - search for an exact frequency
409 * @dev: device for which we do this operation
410 * @freq: frequency to search for
411 * @available: true/false - match for available opp
413 * Return: Searches for exact match in the opp list and returns pointer to the
414 * matching opp if found, else returns ERR_PTR in case of error and should
415 * be handled using IS_ERR. Error return values can be:
416 * EINVAL: for bad pointer
417 * ERANGE: no match found for search
418 * ENODEV: if device not found in list of registered devices
420 * Note: available is a modifier for the search. if available=true, then the
421 * match is for exact matching frequency and is available in the stored OPP
422 * table. if false, the match is for exact frequency which is not available.
424 * This provides a mechanism to enable an opp which is not available currently
425 * or the opposite as well.
427 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
428 * protected pointer. The reason for the same is that the opp pointer which is
429 * returned will remain valid for use with opp_get_{voltage, freq} only while
430 * under the locked area. The pointer returned must be used prior to unlocking
431 * with rcu_read_unlock() to maintain the integrity of the pointer.
433 struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev,
437 struct device_opp *dev_opp;
438 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
440 opp_rcu_lockdep_assert();
442 dev_opp = _find_device_opp(dev);
443 if (IS_ERR(dev_opp)) {
444 int r = PTR_ERR(dev_opp);
445 dev_err(dev, "%s: device OPP not found (%d)\n", __func__, r);
449 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
450 if (temp_opp->available == available &&
451 temp_opp->rate == freq) {
459 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact);
462 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
463 * @dev: device for which we do this operation
464 * @freq: Start frequency
466 * Search for the matching ceil *available* OPP from a starting freq
469 * Return: matching *opp and refreshes *freq accordingly, else returns
470 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
472 * EINVAL: for bad pointer
473 * ERANGE: no match found for search
474 * ENODEV: if device not found in list of registered devices
476 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
477 * protected pointer. The reason for the same is that the opp pointer which is
478 * returned will remain valid for use with opp_get_{voltage, freq} only while
479 * under the locked area. The pointer returned must be used prior to unlocking
480 * with rcu_read_unlock() to maintain the integrity of the pointer.
482 struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev,
485 struct device_opp *dev_opp;
486 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
488 opp_rcu_lockdep_assert();
491 dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq);
492 return ERR_PTR(-EINVAL);
495 dev_opp = _find_device_opp(dev);
497 return ERR_CAST(dev_opp);
499 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
500 if (temp_opp->available && temp_opp->rate >= *freq) {
509 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil);
512 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
513 * @dev: device for which we do this operation
514 * @freq: Start frequency
516 * Search for the matching floor *available* OPP from a starting freq
519 * Return: matching *opp and refreshes *freq accordingly, else returns
520 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
522 * EINVAL: for bad pointer
523 * ERANGE: no match found for search
524 * ENODEV: if device not found in list of registered devices
526 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
527 * protected pointer. The reason for the same is that the opp pointer which is
528 * returned will remain valid for use with opp_get_{voltage, freq} only while
529 * under the locked area. The pointer returned must be used prior to unlocking
530 * with rcu_read_unlock() to maintain the integrity of the pointer.
532 struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev,
535 struct device_opp *dev_opp;
536 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
538 opp_rcu_lockdep_assert();
541 dev_err(dev, "%s: Invalid argument freq=%p\n", __func__, freq);
542 return ERR_PTR(-EINVAL);
545 dev_opp = _find_device_opp(dev);
547 return ERR_CAST(dev_opp);
549 list_for_each_entry_rcu(temp_opp, &dev_opp->opp_list, node) {
550 if (temp_opp->available) {
551 /* go to the next node, before choosing prev */
552 if (temp_opp->rate > *freq)
563 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor);
565 /* List-dev Helpers */
566 static void _kfree_list_dev_rcu(struct rcu_head *head)
568 struct device_list_opp *list_dev;
570 list_dev = container_of(head, struct device_list_opp, rcu_head);
571 kfree_rcu(list_dev, rcu_head);
574 static void _remove_list_dev(struct device_list_opp *list_dev,
575 struct device_opp *dev_opp)
577 list_del(&list_dev->node);
578 call_srcu(&dev_opp->srcu_head.srcu, &list_dev->rcu_head,
579 _kfree_list_dev_rcu);
582 static struct device_list_opp *_add_list_dev(const struct device *dev,
583 struct device_opp *dev_opp)
585 struct device_list_opp *list_dev;
587 list_dev = kzalloc(sizeof(*list_dev), GFP_KERNEL);
591 /* Initialize list-dev */
593 list_add_rcu(&list_dev->node, &dev_opp->dev_list);
599 * _add_device_opp() - Find device OPP table or allocate a new one
600 * @dev: device for which we do this operation
602 * It tries to find an existing table first, if it couldn't find one, it
603 * allocates a new OPP table and returns that.
605 * Return: valid device_opp pointer if success, else NULL.
607 static struct device_opp *_add_device_opp(struct device *dev)
609 struct device_opp *dev_opp;
610 struct device_list_opp *list_dev;
612 /* Check for existing list for 'dev' first */
613 dev_opp = _find_device_opp(dev);
614 if (!IS_ERR(dev_opp))
618 * Allocate a new device OPP table. In the infrequent case where a new
619 * device is needed to be added, we pay this penalty.
621 dev_opp = kzalloc(sizeof(*dev_opp), GFP_KERNEL);
625 INIT_LIST_HEAD(&dev_opp->dev_list);
627 list_dev = _add_list_dev(dev, dev_opp);
633 srcu_init_notifier_head(&dev_opp->srcu_head);
634 INIT_LIST_HEAD(&dev_opp->opp_list);
636 /* Secure the device list modification */
637 list_add_rcu(&dev_opp->node, &dev_opp_list);
642 * _kfree_device_rcu() - Free device_opp RCU handler
645 static void _kfree_device_rcu(struct rcu_head *head)
647 struct device_opp *device_opp = container_of(head, struct device_opp, rcu_head);
649 kfree_rcu(device_opp, rcu_head);
653 * _remove_device_opp() - Removes a device OPP table
654 * @dev_opp: device OPP table to be removed.
656 * Removes/frees device OPP table it it doesn't contain any OPPs.
658 static void _remove_device_opp(struct device_opp *dev_opp)
660 struct device_list_opp *list_dev;
662 if (!list_empty(&dev_opp->opp_list))
665 list_dev = list_first_entry(&dev_opp->dev_list, struct device_list_opp,
668 _remove_list_dev(list_dev, dev_opp);
670 /* dev_list must be empty now */
671 WARN_ON(!list_empty(&dev_opp->dev_list));
673 list_del_rcu(&dev_opp->node);
674 call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head,
679 * _kfree_opp_rcu() - Free OPP RCU handler
682 static void _kfree_opp_rcu(struct rcu_head *head)
684 struct dev_pm_opp *opp = container_of(head, struct dev_pm_opp, rcu_head);
686 kfree_rcu(opp, rcu_head);
690 * _opp_remove() - Remove an OPP from a table definition
691 * @dev_opp: points back to the device_opp struct this opp belongs to
692 * @opp: pointer to the OPP to remove
693 * @notify: OPP_EVENT_REMOVE notification should be sent or not
695 * This function removes an opp definition from the opp list.
697 * Locking: The internal device_opp and opp structures are RCU protected.
698 * It is assumed that the caller holds required mutex for an RCU updater
701 static void _opp_remove(struct device_opp *dev_opp,
702 struct dev_pm_opp *opp, bool notify)
705 * Notify the changes in the availability of the operable
706 * frequency/voltage list.
709 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp);
710 list_del_rcu(&opp->node);
711 call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu);
713 _remove_device_opp(dev_opp);
717 * dev_pm_opp_remove() - Remove an OPP from OPP list
718 * @dev: device for which we do this operation
719 * @freq: OPP to remove with matching 'freq'
721 * This function removes an opp from the opp list.
723 * Locking: The internal device_opp and opp structures are RCU protected.
724 * Hence this function internally uses RCU updater strategy with mutex locks
725 * to keep the integrity of the internal data structures. Callers should ensure
726 * that this function is *NOT* called under RCU protection or in contexts where
727 * mutex cannot be locked.
729 void dev_pm_opp_remove(struct device *dev, unsigned long freq)
731 struct dev_pm_opp *opp;
732 struct device_opp *dev_opp;
735 /* Hold our list modification lock here */
736 mutex_lock(&dev_opp_list_lock);
738 dev_opp = _find_device_opp(dev);
742 list_for_each_entry(opp, &dev_opp->opp_list, node) {
743 if (opp->rate == freq) {
750 dev_warn(dev, "%s: Couldn't find OPP with freq: %lu\n",
755 _opp_remove(dev_opp, opp, true);
757 mutex_unlock(&dev_opp_list_lock);
759 EXPORT_SYMBOL_GPL(dev_pm_opp_remove);
761 static struct dev_pm_opp *_allocate_opp(struct device *dev,
762 struct device_opp **dev_opp)
764 struct dev_pm_opp *opp;
766 /* allocate new OPP node */
767 opp = kzalloc(sizeof(*opp), GFP_KERNEL);
771 INIT_LIST_HEAD(&opp->node);
773 *dev_opp = _add_device_opp(dev);
782 static int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
783 struct device_opp *dev_opp)
785 struct dev_pm_opp *opp;
786 struct list_head *head = &dev_opp->opp_list;
789 * Insert new OPP in order of increasing frequency and discard if
792 * Need to use &dev_opp->opp_list in the condition part of the 'for'
793 * loop, don't replace it with head otherwise it will become an infinite
796 list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) {
797 if (new_opp->rate > opp->rate) {
802 if (new_opp->rate < opp->rate)
806 dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
807 __func__, opp->rate, opp->u_volt, opp->available,
808 new_opp->rate, new_opp->u_volt, new_opp->available);
810 return opp->available && new_opp->u_volt == opp->u_volt ?
814 new_opp->dev_opp = dev_opp;
815 list_add_rcu(&new_opp->node, head);
821 * _opp_add_dynamic() - Allocate a dynamic OPP.
822 * @dev: device for which we do this operation
823 * @freq: Frequency in Hz for this OPP
824 * @u_volt: Voltage in uVolts for this OPP
825 * @dynamic: Dynamically added OPPs.
827 * This function adds an opp definition to the opp list and returns status.
828 * The opp is made available by default and it can be controlled using
829 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
831 * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and
832 * freed by of_free_opp_table.
834 * Locking: The internal device_opp and opp structures are RCU protected.
835 * Hence this function internally uses RCU updater strategy with mutex locks
836 * to keep the integrity of the internal data structures. Callers should ensure
837 * that this function is *NOT* called under RCU protection or in contexts where
838 * mutex cannot be locked.
842 * Duplicate OPPs (both freq and volt are same) and opp->available
843 * -EEXIST Freq are same and volt are different OR
844 * Duplicate OPPs (both freq and volt are same) and !opp->available
845 * -ENOMEM Memory allocation failure
847 static int _opp_add_dynamic(struct device *dev, unsigned long freq,
848 long u_volt, bool dynamic)
850 struct device_opp *dev_opp;
851 struct dev_pm_opp *new_opp;
854 /* Hold our list modification lock here */
855 mutex_lock(&dev_opp_list_lock);
857 new_opp = _allocate_opp(dev, &dev_opp);
863 /* populate the opp table */
864 new_opp->rate = freq;
865 new_opp->u_volt = u_volt;
866 new_opp->available = true;
867 new_opp->dynamic = dynamic;
869 ret = _opp_add(dev, new_opp, dev_opp);
873 mutex_unlock(&dev_opp_list_lock);
876 * Notify the changes in the availability of the operable
877 * frequency/voltage list.
879 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp);
883 _opp_remove(dev_opp, new_opp, false);
885 mutex_unlock(&dev_opp_list_lock);
889 /* TODO: Support multiple regulators */
890 static int opp_get_microvolt(struct dev_pm_opp *opp, struct device *dev)
892 u32 microvolt[3] = {0};
895 /* Missing property isn't a problem, but an invalid entry is */
896 if (!of_find_property(opp->np, "opp-microvolt", NULL))
899 count = of_property_count_u32_elems(opp->np, "opp-microvolt");
901 dev_err(dev, "%s: Invalid opp-microvolt property (%d)\n",
906 /* There can be one or three elements here */
907 if (count != 1 && count != 3) {
908 dev_err(dev, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
913 ret = of_property_read_u32_array(opp->np, "opp-microvolt", microvolt,
916 dev_err(dev, "%s: error parsing opp-microvolt: %d\n", __func__,
921 opp->u_volt = microvolt[0];
922 opp->u_volt_min = microvolt[1];
923 opp->u_volt_max = microvolt[2];
929 * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
930 * @dev: device for which we do this operation
933 * This function adds an opp definition to the opp list and returns status. The
934 * opp can be controlled using dev_pm_opp_enable/disable functions and may be
935 * removed by dev_pm_opp_remove.
937 * Locking: The internal device_opp and opp structures are RCU protected.
938 * Hence this function internally uses RCU updater strategy with mutex locks
939 * to keep the integrity of the internal data structures. Callers should ensure
940 * that this function is *NOT* called under RCU protection or in contexts where
941 * mutex cannot be locked.
945 * Duplicate OPPs (both freq and volt are same) and opp->available
946 * -EEXIST Freq are same and volt are different OR
947 * Duplicate OPPs (both freq and volt are same) and !opp->available
948 * -ENOMEM Memory allocation failure
949 * -EINVAL Failed parsing the OPP node
951 static int _opp_add_static_v2(struct device *dev, struct device_node *np)
953 struct device_opp *dev_opp;
954 struct dev_pm_opp *new_opp;
959 /* Hold our list modification lock here */
960 mutex_lock(&dev_opp_list_lock);
962 new_opp = _allocate_opp(dev, &dev_opp);
968 ret = of_property_read_u64(np, "opp-hz", &rate);
970 dev_err(dev, "%s: opp-hz not found\n", __func__);
975 * Rate is defined as an unsigned long in clk API, and so casting
976 * explicitly to its type. Must be fixed once rate is 64 bit
977 * guaranteed in clk API.
979 new_opp->rate = (unsigned long)rate;
980 new_opp->turbo = of_property_read_bool(np, "turbo-mode");
983 new_opp->dynamic = false;
984 new_opp->available = true;
986 if (!of_property_read_u32(np, "clock-latency-ns", &val))
987 new_opp->clock_latency_ns = val;
989 ret = opp_get_microvolt(new_opp, dev);
993 if (!of_property_read_u32(new_opp->np, "opp-microamp", &val))
994 new_opp->u_amp = val;
996 ret = _opp_add(dev, new_opp, dev_opp);
1000 /* OPP to select on device suspend */
1001 if (of_property_read_bool(np, "opp-suspend")) {
1002 if (dev_opp->suspend_opp)
1003 dev_warn(dev, "%s: Multiple suspend OPPs found (%lu %lu)\n",
1004 __func__, dev_opp->suspend_opp->rate,
1007 dev_opp->suspend_opp = new_opp;
1010 if (new_opp->clock_latency_ns > dev_opp->clock_latency_ns_max)
1011 dev_opp->clock_latency_ns_max = new_opp->clock_latency_ns;
1013 mutex_unlock(&dev_opp_list_lock);
1015 pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
1016 __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt,
1017 new_opp->u_volt_min, new_opp->u_volt_max,
1018 new_opp->clock_latency_ns);
1021 * Notify the changes in the availability of the operable
1022 * frequency/voltage list.
1024 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp);
1028 _opp_remove(dev_opp, new_opp, false);
1030 mutex_unlock(&dev_opp_list_lock);
1035 * dev_pm_opp_add() - Add an OPP table from a table definitions
1036 * @dev: device for which we do this operation
1037 * @freq: Frequency in Hz for this OPP
1038 * @u_volt: Voltage in uVolts for this OPP
1040 * This function adds an opp definition to the opp list and returns status.
1041 * The opp is made available by default and it can be controlled using
1042 * dev_pm_opp_enable/disable functions.
1044 * Locking: The internal device_opp and opp structures are RCU protected.
1045 * Hence this function internally uses RCU updater strategy with mutex locks
1046 * to keep the integrity of the internal data structures. Callers should ensure
1047 * that this function is *NOT* called under RCU protection or in contexts where
1048 * mutex cannot be locked.
1052 * Duplicate OPPs (both freq and volt are same) and opp->available
1053 * -EEXIST Freq are same and volt are different OR
1054 * Duplicate OPPs (both freq and volt are same) and !opp->available
1055 * -ENOMEM Memory allocation failure
1057 int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt)
1059 return _opp_add_dynamic(dev, freq, u_volt, true);
1061 EXPORT_SYMBOL_GPL(dev_pm_opp_add);
1064 * _opp_set_availability() - helper to set the availability of an opp
1065 * @dev: device for which we do this operation
1066 * @freq: OPP frequency to modify availability
1067 * @availability_req: availability status requested for this opp
1069 * Set the availability of an OPP with an RCU operation, opp_{enable,disable}
1070 * share a common logic which is isolated here.
1072 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1073 * copy operation, returns 0 if no modification was done OR modification was
1076 * Locking: The internal device_opp and opp structures are RCU protected.
1077 * Hence this function internally uses RCU updater strategy with mutex locks to
1078 * keep the integrity of the internal data structures. Callers should ensure
1079 * that this function is *NOT* called under RCU protection or in contexts where
1080 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1082 static int _opp_set_availability(struct device *dev, unsigned long freq,
1083 bool availability_req)
1085 struct device_opp *dev_opp;
1086 struct dev_pm_opp *new_opp, *tmp_opp, *opp = ERR_PTR(-ENODEV);
1089 /* keep the node allocated */
1090 new_opp = kmalloc(sizeof(*new_opp), GFP_KERNEL);
1094 mutex_lock(&dev_opp_list_lock);
1096 /* Find the device_opp */
1097 dev_opp = _find_device_opp(dev);
1098 if (IS_ERR(dev_opp)) {
1099 r = PTR_ERR(dev_opp);
1100 dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r);
1104 /* Do we have the frequency? */
1105 list_for_each_entry(tmp_opp, &dev_opp->opp_list, node) {
1106 if (tmp_opp->rate == freq) {
1116 /* Is update really needed? */
1117 if (opp->available == availability_req)
1119 /* copy the old data over */
1122 /* plug in new node */
1123 new_opp->available = availability_req;
1125 list_replace_rcu(&opp->node, &new_opp->node);
1126 mutex_unlock(&dev_opp_list_lock);
1127 call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu);
1129 /* Notify the change of the OPP availability */
1130 if (availability_req)
1131 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ENABLE,
1134 srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_DISABLE,
1140 mutex_unlock(&dev_opp_list_lock);
1146 * dev_pm_opp_enable() - Enable a specific OPP
1147 * @dev: device for which we do this operation
1148 * @freq: OPP frequency to enable
1150 * Enables a provided opp. If the operation is valid, this returns 0, else the
1151 * corresponding error value. It is meant to be used for users an OPP available
1152 * after being temporarily made unavailable with dev_pm_opp_disable.
1154 * Locking: The internal device_opp and opp structures are RCU protected.
1155 * Hence this function indirectly uses RCU and mutex locks to keep the
1156 * integrity of the internal data structures. Callers should ensure that
1157 * this function is *NOT* called under RCU protection or in contexts where
1158 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1160 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1161 * copy operation, returns 0 if no modification was done OR modification was
1164 int dev_pm_opp_enable(struct device *dev, unsigned long freq)
1166 return _opp_set_availability(dev, freq, true);
1168 EXPORT_SYMBOL_GPL(dev_pm_opp_enable);
1171 * dev_pm_opp_disable() - Disable a specific OPP
1172 * @dev: device for which we do this operation
1173 * @freq: OPP frequency to disable
1175 * Disables a provided opp. If the operation is valid, this returns
1176 * 0, else the corresponding error value. It is meant to be a temporary
1177 * control by users to make this OPP not available until the circumstances are
1178 * right to make it available again (with a call to dev_pm_opp_enable).
1180 * Locking: The internal device_opp and opp structures are RCU protected.
1181 * Hence this function indirectly uses RCU and mutex locks to keep the
1182 * integrity of the internal data structures. Callers should ensure that
1183 * this function is *NOT* called under RCU protection or in contexts where
1184 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1186 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1187 * copy operation, returns 0 if no modification was done OR modification was
1190 int dev_pm_opp_disable(struct device *dev, unsigned long freq)
1192 return _opp_set_availability(dev, freq, false);
1194 EXPORT_SYMBOL_GPL(dev_pm_opp_disable);
1197 * dev_pm_opp_get_notifier() - find notifier_head of the device with opp
1198 * @dev: device pointer used to lookup device OPPs.
1200 * Return: pointer to notifier head if found, otherwise -ENODEV or
1201 * -EINVAL based on type of error casted as pointer. value must be checked
1202 * with IS_ERR to determine valid pointer or error result.
1204 * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU
1205 * protected pointer. The reason for the same is that the opp pointer which is
1206 * returned will remain valid for use with opp_get_{voltage, freq} only while
1207 * under the locked area. The pointer returned must be used prior to unlocking
1208 * with rcu_read_unlock() to maintain the integrity of the pointer.
1210 struct srcu_notifier_head *dev_pm_opp_get_notifier(struct device *dev)
1212 struct device_opp *dev_opp = _find_device_opp(dev);
1214 if (IS_ERR(dev_opp))
1215 return ERR_CAST(dev_opp); /* matching type */
1217 return &dev_opp->srcu_head;
1219 EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier);
1223 * of_free_opp_table() - Free OPP table entries created from static DT entries
1224 * @dev: device pointer used to lookup device OPPs.
1226 * Free OPPs created using static entries present in DT.
1228 * Locking: The internal device_opp and opp structures are RCU protected.
1229 * Hence this function indirectly uses RCU updater strategy with mutex locks
1230 * to keep the integrity of the internal data structures. Callers should ensure
1231 * that this function is *NOT* called under RCU protection or in contexts where
1232 * mutex cannot be locked.
1234 void of_free_opp_table(struct device *dev)
1236 struct device_opp *dev_opp;
1237 struct dev_pm_opp *opp, *tmp;
1239 /* Hold our list modification lock here */
1240 mutex_lock(&dev_opp_list_lock);
1242 /* Check for existing list for 'dev' */
1243 dev_opp = _find_device_opp(dev);
1244 if (IS_ERR(dev_opp)) {
1245 int error = PTR_ERR(dev_opp);
1247 if (error != -ENODEV)
1248 WARN(1, "%s: dev_opp: %d\n",
1249 IS_ERR_OR_NULL(dev) ?
1250 "Invalid device" : dev_name(dev),
1255 /* Find if dev_opp manages a single device */
1256 if (list_is_singular(&dev_opp->dev_list)) {
1257 /* Free static OPPs */
1258 list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) {
1260 _opp_remove(dev_opp, opp, true);
1263 _remove_list_dev(_find_list_dev(dev, dev_opp), dev_opp);
1267 mutex_unlock(&dev_opp_list_lock);
1269 EXPORT_SYMBOL_GPL(of_free_opp_table);
1271 void of_cpumask_free_opp_table(cpumask_var_t cpumask)
1273 struct device *cpu_dev;
1276 WARN_ON(cpumask_empty(cpumask));
1278 for_each_cpu(cpu, cpumask) {
1279 cpu_dev = get_cpu_device(cpu);
1281 pr_err("%s: failed to get cpu%d device\n", __func__,
1286 of_free_opp_table(cpu_dev);
1289 EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table);
1291 /* Returns opp descriptor node from its phandle. Caller must do of_node_put() */
1292 static struct device_node *
1293 _of_get_opp_desc_node_from_prop(struct device *dev, const struct property *prop)
1295 struct device_node *opp_np;
1297 opp_np = of_find_node_by_phandle(be32_to_cpup(prop->value));
1299 dev_err(dev, "%s: Prop: %s contains invalid opp desc phandle\n",
1300 __func__, prop->name);
1301 return ERR_PTR(-EINVAL);
1307 /* Returns opp descriptor node for a device. Caller must do of_node_put() */
1308 static struct device_node *_of_get_opp_desc_node(struct device *dev)
1310 const struct property *prop;
1312 prop = of_find_property(dev->of_node, "operating-points-v2", NULL);
1314 return ERR_PTR(-ENODEV);
1316 return ERR_PTR(-ENODATA);
1319 * TODO: Support for multiple OPP tables.
1321 * There should be only ONE phandle present in "operating-points-v2"
1324 if (prop->length != sizeof(__be32)) {
1325 dev_err(dev, "%s: Invalid opp desc phandle\n", __func__);
1326 return ERR_PTR(-EINVAL);
1329 return _of_get_opp_desc_node_from_prop(dev, prop);
1332 /* Initializes OPP tables based on new bindings */
1333 static int _of_init_opp_table_v2(struct device *dev,
1334 const struct property *prop)
1336 struct device_node *opp_np, *np;
1337 struct device_opp *dev_opp;
1338 int ret = 0, count = 0;
1344 opp_np = _of_get_opp_desc_node_from_prop(dev, prop);
1346 return PTR_ERR(opp_np);
1348 dev_opp = _managed_opp(opp_np);
1350 /* OPPs are already managed */
1351 if (!_add_list_dev(dev, dev_opp))
1356 /* We have opp-list node now, iterate over it and add OPPs */
1357 for_each_available_child_of_node(opp_np, np) {
1360 ret = _opp_add_static_v2(dev, np);
1362 dev_err(dev, "%s: Failed to add OPP, %d\n", __func__,
1368 /* There should be one of more OPP defined */
1369 if (WARN_ON(!count)) {
1374 dev_opp = _find_device_opp(dev);
1375 if (WARN_ON(IS_ERR(dev_opp))) {
1376 ret = PTR_ERR(dev_opp);
1380 dev_opp->np = opp_np;
1381 dev_opp->shared_opp = of_property_read_bool(opp_np, "opp-shared");
1383 of_node_put(opp_np);
1387 of_free_opp_table(dev);
1389 of_node_put(opp_np);
1394 /* Initializes OPP tables based on old-deprecated bindings */
1395 static int _of_init_opp_table_v1(struct device *dev)
1397 const struct property *prop;
1401 prop = of_find_property(dev->of_node, "operating-points", NULL);
1408 * Each OPP is a set of tuples consisting of frequency and
1409 * voltage like <freq-kHz vol-uV>.
1411 nr = prop->length / sizeof(u32);
1413 dev_err(dev, "%s: Invalid OPP list\n", __func__);
1419 unsigned long freq = be32_to_cpup(val++) * 1000;
1420 unsigned long volt = be32_to_cpup(val++);
1422 if (_opp_add_dynamic(dev, freq, volt, false))
1423 dev_warn(dev, "%s: Failed to add OPP %ld\n",
1432 * of_init_opp_table() - Initialize opp table from device tree
1433 * @dev: device pointer used to lookup device OPPs.
1435 * Register the initial OPP table with the OPP library for given device.
1437 * Locking: The internal device_opp and opp structures are RCU protected.
1438 * Hence this function indirectly uses RCU updater strategy with mutex locks
1439 * to keep the integrity of the internal data structures. Callers should ensure
1440 * that this function is *NOT* called under RCU protection or in contexts where
1441 * mutex cannot be locked.
1445 * Duplicate OPPs (both freq and volt are same) and opp->available
1446 * -EEXIST Freq are same and volt are different OR
1447 * Duplicate OPPs (both freq and volt are same) and !opp->available
1448 * -ENOMEM Memory allocation failure
1449 * -ENODEV when 'operating-points' property is not found or is invalid data
1451 * -ENODATA when empty 'operating-points' property is found
1452 * -EINVAL when invalid entries are found in opp-v2 table
1454 int of_init_opp_table(struct device *dev)
1456 const struct property *prop;
1459 * OPPs have two version of bindings now. The older one is deprecated,
1460 * try for the new binding first.
1462 prop = of_find_property(dev->of_node, "operating-points-v2", NULL);
1465 * Try old-deprecated bindings for backward compatibility with
1468 return _of_init_opp_table_v1(dev);
1471 return _of_init_opp_table_v2(dev, prop);
1473 EXPORT_SYMBOL_GPL(of_init_opp_table);
1475 int of_cpumask_init_opp_table(cpumask_var_t cpumask)
1477 struct device *cpu_dev;
1480 WARN_ON(cpumask_empty(cpumask));
1482 for_each_cpu(cpu, cpumask) {
1483 cpu_dev = get_cpu_device(cpu);
1485 pr_err("%s: failed to get cpu%d device\n", __func__,
1490 ret = of_init_opp_table(cpu_dev);
1492 pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
1493 __func__, cpu, ret);
1495 /* Free all other OPPs */
1496 of_cpumask_free_opp_table(cpumask);
1503 EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table);
1505 /* Required only for V1 bindings, as v2 can manage it from DT itself */
1506 int set_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask)
1508 struct device_list_opp *list_dev;
1509 struct device_opp *dev_opp;
1515 dev_opp = _find_device_opp(cpu_dev);
1516 if (IS_ERR(dev_opp)) {
1518 goto out_rcu_read_unlock;
1521 for_each_cpu(cpu, cpumask) {
1522 if (cpu == cpu_dev->id)
1525 dev = get_cpu_device(cpu);
1527 dev_err(cpu_dev, "%s: failed to get cpu%d device\n",
1532 list_dev = _add_list_dev(dev, dev_opp);
1534 dev_err(dev, "%s: failed to add list-dev for cpu%d device\n",
1539 out_rcu_read_unlock:
1544 EXPORT_SYMBOL_GPL(set_cpus_sharing_opps);
1547 * Works only for OPP v2 bindings.
1549 * cpumask should be already set to mask of cpu_dev->id.
1550 * Returns -ENOENT if operating-points-v2 bindings aren't supported.
1552 int of_get_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask)
1554 struct device_node *np, *tmp_np;
1555 struct device *tcpu_dev;
1558 /* Get OPP descriptor node */
1559 np = _of_get_opp_desc_node(cpu_dev);
1561 dev_dbg(cpu_dev, "%s: Couldn't find opp node: %ld\n", __func__,
1566 /* OPPs are shared ? */
1567 if (!of_property_read_bool(np, "opp-shared"))
1570 for_each_possible_cpu(cpu) {
1571 if (cpu == cpu_dev->id)
1574 tcpu_dev = get_cpu_device(cpu);
1576 dev_err(cpu_dev, "%s: failed to get cpu%d device\n",
1582 /* Get OPP descriptor node */
1583 tmp_np = _of_get_opp_desc_node(tcpu_dev);
1584 if (IS_ERR(tmp_np)) {
1585 dev_err(tcpu_dev, "%s: Couldn't find opp node: %ld\n",
1586 __func__, PTR_ERR(tmp_np));
1587 ret = PTR_ERR(tmp_np);
1591 /* CPUs are sharing opp node */
1593 cpumask_set_cpu(cpu, cpumask);
1595 of_node_put(tmp_np);
1602 EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps);