Merge branch 'libnvdimm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdim...

[karo-tx-linux.git] / kernel / rcu / rcu.h

diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 87326479b39ac19102ccd04bc31c0b6828cf121b..73e16ec4054b83d163f58d6786df4e5fb6a00ae4 100644 (file)
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -61,20 +61,51 @@
  * Grace-period counter management.
  */
 
+#define RCU_SEQ_CTR_SHIFT      2
+#define RCU_SEQ_STATE_MASK     ((1 << RCU_SEQ_CTR_SHIFT) - 1)
+
+/*
+ * Return the counter portion of a sequence number previously returned
+ * by rcu_seq_snap() or rcu_seq_current().
+ */
+static inline unsigned long rcu_seq_ctr(unsigned long s)
+{
+       return s >> RCU_SEQ_CTR_SHIFT;
+}
+
+/*
+ * Return the state portion of a sequence number previously returned
+ * by rcu_seq_snap() or rcu_seq_current().
+ */
+static inline int rcu_seq_state(unsigned long s)
+{
+       return s & RCU_SEQ_STATE_MASK;
+}
+
+/*
+ * Set the state portion of the pointed-to sequence number.
+ * The caller is responsible for preventing conflicting updates.
+ */
+static inline void rcu_seq_set_state(unsigned long *sp, int newstate)
+{
+       WARN_ON_ONCE(newstate & ~RCU_SEQ_STATE_MASK);
+       WRITE_ONCE(*sp, (*sp & ~RCU_SEQ_STATE_MASK) + newstate);
+}
+
 /* Adjust sequence number for start of update-side operation. */
 static inline void rcu_seq_start(unsigned long *sp)
 {
        WRITE_ONCE(*sp, *sp + 1);
        smp_mb(); /* Ensure update-side operation after counter increment. */
-       WARN_ON_ONCE(!(*sp & 0x1));
+       WARN_ON_ONCE(rcu_seq_state(*sp) != 1);
 }
 
 /* Adjust sequence number for end of update-side operation. */
 static inline void rcu_seq_end(unsigned long *sp)
 {
        smp_mb(); /* Ensure update-side operation before counter increment. */
-       WARN_ON_ONCE(!(*sp & 0x1));
-       WRITE_ONCE(*sp, *sp + 1);
+       WARN_ON_ONCE(!rcu_seq_state(*sp));
+       WRITE_ONCE(*sp, (*sp | RCU_SEQ_STATE_MASK) + 1);
 }
 
 /* Take a snapshot of the update side's sequence number. */
@@ -82,7 +113,7 @@ static inline unsigned long rcu_seq_snap(unsigned long *sp)
 {
        unsigned long s;
 
-       s = (READ_ONCE(*sp) + 3) & ~0x1;
+       s = (READ_ONCE(*sp) + 2 * RCU_SEQ_STATE_MASK + 1) & ~RCU_SEQ_STATE_MASK;
        smp_mb(); /* Above access must not bleed into critical section. */
        return s;
 }
@@ -195,6 +226,7 @@ extern void resched_cpu(int cpu);
 #include <linux/rcu_node_tree.h>
 
 extern int rcu_num_lvls;
+extern int num_rcu_lvl[];
 extern int rcu_num_nodes;
 static bool rcu_fanout_exact;
 static int rcu_fanout_leaf;
@@ -224,6 +256,41 @@ static inline void rcu_init_levelspread(int *levelspread, const int *levelcnt)
        }
 }
 
+/*
+ * Do a full breadth-first scan of the rcu_node structures for the
+ * specified rcu_state structure.
+ */
+#define rcu_for_each_node_breadth_first(rsp, rnp) \
+       for ((rnp) = &(rsp)->node[0]; \
+            (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
+
+/*
+ * Do a breadth-first scan of the non-leaf rcu_node structures for the
+ * specified rcu_state structure.  Note that if there is a singleton
+ * rcu_node tree with but one rcu_node structure, this loop is a no-op.
+ */
+#define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \
+       for ((rnp) = &(rsp)->node[0]; \
+            (rnp) < (rsp)->level[rcu_num_lvls - 1]; (rnp)++)
+
+/*
+ * Scan the leaves of the rcu_node hierarchy for the specified rcu_state
+ * structure.  Note that if there is a singleton rcu_node tree with but
+ * one rcu_node structure, this loop -will- visit the rcu_node structure.
+ * It is still a leaf node, even if it is also the root node.
+ */
+#define rcu_for_each_leaf_node(rsp, rnp) \
+       for ((rnp) = (rsp)->level[rcu_num_lvls - 1]; \
+            (rnp) < &(rsp)->node[rcu_num_nodes]; (rnp)++)
+
+/*
+ * Iterate over all possible CPUs in a leaf RCU node.
+ */
+#define for_each_leaf_node_possible_cpu(rnp, cpu) \
+       for ((cpu) = cpumask_next(rnp->grplo - 1, cpu_possible_mask); \
+            cpu <= rnp->grphi; \
+            cpu = cpumask_next((cpu), cpu_possible_mask))
+
 #endif /* #if defined(SRCU) || !defined(TINY_RCU) */
 
 #endif /* __LINUX_RCU_H */