2 * Multipath support for RPC
4 * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
6 * Trond Myklebust <trond.myklebust@primarydata.com>
9 #include <linux/types.h>
10 #include <linux/kref.h>
11 #include <linux/list.h>
12 #include <linux/rcupdate.h>
13 #include <linux/rculist.h>
14 #include <linux/slab.h>
15 #include <asm/cmpxchg.h>
16 #include <linux/spinlock.h>
17 #include <linux/sunrpc/xprt.h>
18 #include <linux/sunrpc/xprtmultipath.h>
20 typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
21 const struct rpc_xprt *cur);
23 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
24 static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
25 static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
27 static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
28 struct rpc_xprt *xprt)
30 if (unlikely(xprt_get(xprt) == NULL))
32 list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
34 if (xps->xps_nxprts == 0)
35 xps->xps_net = xprt->xprt_net;
40 * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
41 * @xps: pointer to struct rpc_xprt_switch
42 * @xprt: pointer to struct rpc_xprt
44 * Adds xprt to the end of the list of struct rpc_xprt in xps.
46 void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
47 struct rpc_xprt *xprt)
51 spin_lock(&xps->xps_lock);
52 if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL)
53 xprt_switch_add_xprt_locked(xps, xprt);
54 spin_unlock(&xps->xps_lock);
57 static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
58 struct rpc_xprt *xprt)
60 if (unlikely(xprt == NULL))
63 if (xps->xps_nxprts == 0)
66 list_del_rcu(&xprt->xprt_switch);
70 * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
71 * @xps: pointer to struct rpc_xprt_switch
72 * @xprt: pointer to struct rpc_xprt
74 * Removes xprt from the list of struct rpc_xprt in xps.
76 void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
77 struct rpc_xprt *xprt)
79 spin_lock(&xps->xps_lock);
80 xprt_switch_remove_xprt_locked(xps, xprt);
81 spin_unlock(&xps->xps_lock);
86 * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
87 * @xprt: pointer to struct rpc_xprt
88 * @gfp_flags: allocation flags
90 * On success, returns an initialised struct rpc_xprt_switch, containing
91 * the entry xprt. Returns NULL on failure.
93 struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
96 struct rpc_xprt_switch *xps;
98 xps = kmalloc(sizeof(*xps), gfp_flags);
100 spin_lock_init(&xps->xps_lock);
101 kref_init(&xps->xps_kref);
103 INIT_LIST_HEAD(&xps->xps_xprt_list);
104 xps->xps_iter_ops = &rpc_xprt_iter_singular;
105 xprt_switch_add_xprt_locked(xps, xprt);
111 static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
113 spin_lock(&xps->xps_lock);
114 while (!list_empty(&xps->xps_xprt_list)) {
115 struct rpc_xprt *xprt;
117 xprt = list_first_entry(&xps->xps_xprt_list,
118 struct rpc_xprt, xprt_switch);
119 xprt_switch_remove_xprt_locked(xps, xprt);
120 spin_unlock(&xps->xps_lock);
122 spin_lock(&xps->xps_lock);
124 spin_unlock(&xps->xps_lock);
127 static void xprt_switch_free(struct kref *kref)
129 struct rpc_xprt_switch *xps = container_of(kref,
130 struct rpc_xprt_switch, xps_kref);
132 xprt_switch_free_entries(xps);
133 kfree_rcu(xps, xps_rcu);
137 * xprt_switch_get - Return a reference to a rpc_xprt_switch
138 * @xps: pointer to struct rpc_xprt_switch
140 * Returns a reference to xps unless the refcount is already zero.
142 struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
144 if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
150 * xprt_switch_put - Release a reference to a rpc_xprt_switch
151 * @xps: pointer to struct rpc_xprt_switch
153 * Release the reference to xps, and free it once the refcount is zero.
155 void xprt_switch_put(struct rpc_xprt_switch *xps)
158 kref_put(&xps->xps_kref, xprt_switch_free);
162 * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
163 * @xps: pointer to struct rpc_xprt_switch
165 * Sets a round-robin default policy for iterators acting on xps.
167 void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
169 if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
170 WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
174 const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
176 if (xpi->xpi_ops != NULL)
178 return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
182 void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
187 void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
189 WRITE_ONCE(xpi->xpi_cursor, NULL);
193 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
195 return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch);
199 struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
201 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
205 return xprt_switch_find_first_entry(&xps->xps_xprt_list);
209 struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
210 const struct rpc_xprt *cur)
212 struct rpc_xprt *pos;
214 list_for_each_entry_rcu(pos, head, xprt_switch) {
222 struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
224 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
225 struct list_head *head;
229 head = &xps->xps_xprt_list;
230 if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
231 return xprt_switch_find_first_entry(head);
232 return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
236 struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
237 const struct rpc_xprt *cur)
239 struct rpc_xprt *pos, *prev = NULL;
241 list_for_each_entry_rcu(pos, head, xprt_switch) {
250 struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head,
251 struct rpc_xprt **cursor,
252 xprt_switch_find_xprt_t find_next)
254 struct rpc_xprt *cur, *pos, *old;
256 cur = READ_ONCE(*cursor);
259 pos = find_next(head, old);
262 cur = cmpxchg_relaxed(cursor, old, pos);
270 struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
271 xprt_switch_find_xprt_t find_next)
273 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
274 struct list_head *head;
278 head = &xps->xps_xprt_list;
279 if (xps->xps_nxprts < 2)
280 return xprt_switch_find_first_entry(head);
281 return xprt_switch_set_next_cursor(head, &xpi->xpi_cursor, find_next);
285 struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head,
286 const struct rpc_xprt *cur)
288 struct rpc_xprt *ret;
290 ret = xprt_switch_find_next_entry(head, cur);
293 return xprt_switch_find_first_entry(head);
297 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
299 return xprt_iter_next_entry_multiple(xpi,
300 xprt_switch_find_next_entry_roundrobin);
304 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
306 return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry);
310 * xprt_iter_rewind - Resets the xprt iterator
311 * @xpi: pointer to rpc_xprt_iter
313 * Resets xpi to ensure that it points to the first entry in the list
317 void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
320 xprt_iter_ops(xpi)->xpi_rewind(xpi);
324 static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
325 struct rpc_xprt_switch *xps,
326 const struct rpc_xprt_iter_ops *ops)
328 rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
329 xpi->xpi_cursor = NULL;
334 * xprt_iter_init - Initialise an xprt iterator
335 * @xpi: pointer to rpc_xprt_iter
336 * @xps: pointer to rpc_xprt_switch
338 * Initialises the iterator to use the default iterator ops
339 * as set in xps. This function is mainly intended for internal
340 * use in the rpc_client.
342 void xprt_iter_init(struct rpc_xprt_iter *xpi,
343 struct rpc_xprt_switch *xps)
345 __xprt_iter_init(xpi, xps, NULL);
349 * xprt_iter_init_listall - Initialise an xprt iterator
350 * @xpi: pointer to rpc_xprt_iter
351 * @xps: pointer to rpc_xprt_switch
353 * Initialises the iterator to iterate once through the entire list
356 void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
357 struct rpc_xprt_switch *xps)
359 __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
363 * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
364 * @xpi: pointer to rpc_xprt_iter
365 * @xps: pointer to a new rpc_xprt_switch or NULL
367 * Swaps out the existing xpi->xpi_xpswitch with a new value.
369 struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
370 struct rpc_xprt_switch *newswitch)
372 struct rpc_xprt_switch __rcu *oldswitch;
374 /* Atomically swap out the old xpswitch */
375 oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
376 if (newswitch != NULL)
377 xprt_iter_rewind(xpi);
378 return rcu_dereference_protected(oldswitch, true);
382 * xprt_iter_destroy - Destroys the xprt iterator
383 * @xpi pointer to rpc_xprt_iter
385 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
387 xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
391 * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
392 * @xpi: pointer to rpc_xprt_iter
394 * Returns a pointer to the struct rpc_xprt that is currently
395 * pointed to by the cursor.
396 * Caller must be holding rcu_read_lock().
398 struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
400 WARN_ON_ONCE(!rcu_read_lock_held());
401 return xprt_iter_ops(xpi)->xpi_xprt(xpi);
405 struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
406 struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
408 struct rpc_xprt *ret;
415 } while (ret == NULL);
420 * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
421 * @xpi: pointer to rpc_xprt_iter
423 * Returns a reference to the struct rpc_xprt that is currently
424 * pointed to by the cursor.
426 struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
428 struct rpc_xprt *xprt;
431 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
437 * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
438 * @xpi: pointer to rpc_xprt_iter
440 * Returns a reference to the struct rpc_xprt that immediately follows the
441 * entry pointed to by the cursor.
443 struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
445 struct rpc_xprt *xprt;
448 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
453 /* Policy for always returning the first entry in the rpc_xprt_switch */
455 const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
456 .xpi_rewind = xprt_iter_no_rewind,
457 .xpi_xprt = xprt_iter_first_entry,
458 .xpi_next = xprt_iter_first_entry,
461 /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
463 const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
464 .xpi_rewind = xprt_iter_default_rewind,
465 .xpi_xprt = xprt_iter_current_entry,
466 .xpi_next = xprt_iter_next_entry_roundrobin,
469 /* Policy for once-through iteration of entries in the rpc_xprt_switch */
471 const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
472 .xpi_rewind = xprt_iter_default_rewind,
473 .xpi_xprt = xprt_iter_current_entry,
474 .xpi_next = xprt_iter_next_entry_all,