4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2011, 2012, Intel Corporation.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * libcfs/libcfs/workitem.c
38 * Author: Isaac Huang <isaac@clusterfs.com>
39 * Liang Zhen <zhen.liang@sun.com>
42 #define DEBUG_SUBSYSTEM S_LNET
44 #include "../../include/linux/libcfs/libcfs.h"
46 #define CFS_WS_NAME_LEN 16
49 struct list_head ws_list; /* chain on global list */
50 /** serialised workitems */
52 /** where schedulers sleep */
53 wait_queue_head_t ws_waitq;
54 /** concurrent workitems */
55 struct list_head ws_runq;
56 /** rescheduled running-workitems, a workitem can be rescheduled
57 * while running in wi_action(), but we don't to execute it again
58 * unless it returns from wi_action(), so we put it on ws_rerunq
59 * while rescheduling, and move it to runq after it returns
61 struct list_head ws_rerunq;
62 /** CPT-table for this scheduler */
63 struct cfs_cpt_table *ws_cptab;
64 /** CPT id for affinity */
66 /** number of scheduled workitems */
68 /** started scheduler thread, protected by cfs_wi_data::wi_glock */
69 unsigned int ws_nthreads:30;
70 /** shutting down, protected by cfs_wi_data::wi_glock */
71 unsigned int ws_stopping:1;
72 /** serialize starting thread, protected by cfs_wi_data::wi_glock */
73 unsigned int ws_starting:1;
75 char ws_name[CFS_WS_NAME_LEN];
78 static struct cfs_workitem_data {
81 /** list of all schedulers */
82 struct list_head wi_scheds;
83 /** WI module is initialized */
85 /** shutting down the whole WI module */
90 cfs_wi_sched_cansleep(struct cfs_wi_sched *sched)
92 spin_lock(&sched->ws_lock);
93 if (sched->ws_stopping) {
94 spin_unlock(&sched->ws_lock);
98 if (!list_empty(&sched->ws_runq)) {
99 spin_unlock(&sched->ws_lock);
102 spin_unlock(&sched->ws_lock);
107 * 0. it only works when called from wi->wi_action.
108 * 1. when it returns no one shall try to schedule the workitem.
111 cfs_wi_exit(struct cfs_wi_sched *sched, cfs_workitem_t *wi)
113 LASSERT(!in_interrupt()); /* because we use plain spinlock */
114 LASSERT(!sched->ws_stopping);
116 spin_lock(&sched->ws_lock);
118 LASSERT(wi->wi_running);
119 if (wi->wi_scheduled) { /* cancel pending schedules */
120 LASSERT(!list_empty(&wi->wi_list));
121 list_del_init(&wi->wi_list);
123 LASSERT(sched->ws_nscheduled > 0);
124 sched->ws_nscheduled--;
127 LASSERT(list_empty(&wi->wi_list));
129 wi->wi_scheduled = 1; /* LBUG future schedule attempts */
130 spin_unlock(&sched->ws_lock);
134 EXPORT_SYMBOL(cfs_wi_exit);
137 * cancel schedule request of workitem \a wi
140 cfs_wi_deschedule(struct cfs_wi_sched *sched, cfs_workitem_t *wi)
144 LASSERT(!in_interrupt()); /* because we use plain spinlock */
145 LASSERT(!sched->ws_stopping);
148 * return 0 if it's running already, otherwise return 1, which
149 * means the workitem will not be scheduled and will not have
150 * any race with wi_action.
152 spin_lock(&sched->ws_lock);
154 rc = !(wi->wi_running);
156 if (wi->wi_scheduled) { /* cancel pending schedules */
157 LASSERT(!list_empty(&wi->wi_list));
158 list_del_init(&wi->wi_list);
160 LASSERT(sched->ws_nscheduled > 0);
161 sched->ws_nscheduled--;
163 wi->wi_scheduled = 0;
166 LASSERT (list_empty(&wi->wi_list));
168 spin_unlock(&sched->ws_lock);
171 EXPORT_SYMBOL(cfs_wi_deschedule);
174 * Workitem scheduled with (serial == 1) is strictly serialised not only with
175 * itself, but also with others scheduled this way.
177 * Now there's only one static serialised queue, but in the future more might
178 * be added, and even dynamic creation of serialised queues might be supported.
181 cfs_wi_schedule(struct cfs_wi_sched *sched, cfs_workitem_t *wi)
183 LASSERT(!in_interrupt()); /* because we use plain spinlock */
184 LASSERT(!sched->ws_stopping);
186 spin_lock(&sched->ws_lock);
188 if (!wi->wi_scheduled) {
189 LASSERT (list_empty(&wi->wi_list));
191 wi->wi_scheduled = 1;
192 sched->ws_nscheduled++;
193 if (!wi->wi_running) {
194 list_add_tail(&wi->wi_list, &sched->ws_runq);
195 wake_up(&sched->ws_waitq);
197 list_add(&wi->wi_list, &sched->ws_rerunq);
201 LASSERT (!list_empty(&wi->wi_list));
202 spin_unlock(&sched->ws_lock);
205 EXPORT_SYMBOL(cfs_wi_schedule);
208 cfs_wi_scheduler (void *arg)
210 struct cfs_wi_sched *sched = (struct cfs_wi_sched *)arg;
214 /* CPT affinity scheduler? */
216 if (cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt) != 0)
217 CWARN("Failed to bind %s on CPT %d\n",
218 sched->ws_name, sched->ws_cpt);
220 spin_lock(&cfs_wi_data.wi_glock);
222 LASSERT(sched->ws_starting == 1);
223 sched->ws_starting--;
224 sched->ws_nthreads++;
226 spin_unlock(&cfs_wi_data.wi_glock);
228 spin_lock(&sched->ws_lock);
230 while (!sched->ws_stopping) {
235 while (!list_empty(&sched->ws_runq) &&
236 nloops < CFS_WI_RESCHED) {
237 wi = list_entry(sched->ws_runq.next, cfs_workitem_t,
239 LASSERT(wi->wi_scheduled && !wi->wi_running);
241 list_del_init(&wi->wi_list);
243 LASSERT(sched->ws_nscheduled > 0);
244 sched->ws_nscheduled--;
247 wi->wi_scheduled = 0;
249 spin_unlock(&sched->ws_lock);
252 rc = (*wi->wi_action) (wi);
254 spin_lock(&sched->ws_lock);
255 if (rc != 0) /* WI should be dead, even be freed! */
259 if (list_empty(&wi->wi_list))
262 LASSERT(wi->wi_scheduled);
263 /* wi is rescheduled, should be on rerunq now, we
264 * move it to runq so it can run action now */
265 list_move_tail(&wi->wi_list, &sched->ws_runq);
268 if (!list_empty(&sched->ws_runq)) {
269 spin_unlock(&sched->ws_lock);
270 /* don't sleep because some workitems still
271 * expect me to come back soon */
273 spin_lock(&sched->ws_lock);
277 spin_unlock(&sched->ws_lock);
278 rc = wait_event_interruptible_exclusive(sched->ws_waitq,
279 !cfs_wi_sched_cansleep(sched));
280 spin_lock(&sched->ws_lock);
283 spin_unlock(&sched->ws_lock);
285 spin_lock(&cfs_wi_data.wi_glock);
286 sched->ws_nthreads--;
287 spin_unlock(&cfs_wi_data.wi_glock);
293 cfs_wi_sched_destroy(struct cfs_wi_sched *sched)
297 LASSERT(cfs_wi_data.wi_init);
298 LASSERT(!cfs_wi_data.wi_stopping);
300 spin_lock(&cfs_wi_data.wi_glock);
301 if (sched->ws_stopping) {
302 CDEBUG(D_INFO, "%s is in progress of stopping\n",
304 spin_unlock(&cfs_wi_data.wi_glock);
308 LASSERT(!list_empty(&sched->ws_list));
309 sched->ws_stopping = 1;
311 spin_unlock(&cfs_wi_data.wi_glock);
314 wake_up_all(&sched->ws_waitq);
316 spin_lock(&cfs_wi_data.wi_glock);
317 while (sched->ws_nthreads > 0) {
318 CDEBUG(is_power_of_2(++i) ? D_WARNING : D_NET,
319 "waiting for %d threads of WI sched[%s] to terminate\n",
320 sched->ws_nthreads, sched->ws_name);
322 spin_unlock(&cfs_wi_data.wi_glock);
323 set_current_state(TASK_UNINTERRUPTIBLE);
324 schedule_timeout(cfs_time_seconds(1) / 20);
325 spin_lock(&cfs_wi_data.wi_glock);
328 list_del(&sched->ws_list);
330 spin_unlock(&cfs_wi_data.wi_glock);
331 LASSERT(sched->ws_nscheduled == 0);
333 LIBCFS_FREE(sched, sizeof(*sched));
335 EXPORT_SYMBOL(cfs_wi_sched_destroy);
338 cfs_wi_sched_create(char *name, struct cfs_cpt_table *cptab,
339 int cpt, int nthrs, struct cfs_wi_sched **sched_pp)
341 struct cfs_wi_sched *sched;
344 LASSERT(cfs_wi_data.wi_init);
345 LASSERT(!cfs_wi_data.wi_stopping);
346 LASSERT(!cptab || cpt == CFS_CPT_ANY ||
347 (cpt >= 0 && cpt < cfs_cpt_number(cptab)));
349 LIBCFS_ALLOC(sched, sizeof(*sched));
353 strlcpy(sched->ws_name, name, CFS_WS_NAME_LEN);
355 sched->ws_cptab = cptab;
358 spin_lock_init(&sched->ws_lock);
359 init_waitqueue_head(&sched->ws_waitq);
360 INIT_LIST_HEAD(&sched->ws_runq);
361 INIT_LIST_HEAD(&sched->ws_rerunq);
362 INIT_LIST_HEAD(&sched->ws_list);
367 struct task_struct *task;
369 spin_lock(&cfs_wi_data.wi_glock);
370 while (sched->ws_starting > 0) {
371 spin_unlock(&cfs_wi_data.wi_glock);
373 spin_lock(&cfs_wi_data.wi_glock);
376 sched->ws_starting++;
377 spin_unlock(&cfs_wi_data.wi_glock);
379 if (sched->ws_cptab && sched->ws_cpt >= 0) {
380 snprintf(name, sizeof(name), "%s_%02d_%02u",
381 sched->ws_name, sched->ws_cpt,
384 snprintf(name, sizeof(name), "%s_%02u",
385 sched->ws_name, sched->ws_nthreads);
388 task = kthread_run(cfs_wi_scheduler, sched, "%s", name);
395 CERROR("Failed to create thread for WI scheduler %s: %d\n",
398 spin_lock(&cfs_wi_data.wi_glock);
400 /* make up for cfs_wi_sched_destroy */
401 list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
402 sched->ws_starting--;
404 spin_unlock(&cfs_wi_data.wi_glock);
406 cfs_wi_sched_destroy(sched);
409 spin_lock(&cfs_wi_data.wi_glock);
410 list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
411 spin_unlock(&cfs_wi_data.wi_glock);
416 EXPORT_SYMBOL(cfs_wi_sched_create);
421 memset(&cfs_wi_data, 0, sizeof(cfs_wi_data));
423 spin_lock_init(&cfs_wi_data.wi_glock);
424 INIT_LIST_HEAD(&cfs_wi_data.wi_scheds);
425 cfs_wi_data.wi_init = 1;
431 cfs_wi_shutdown(void)
433 struct cfs_wi_sched *sched;
435 spin_lock(&cfs_wi_data.wi_glock);
436 cfs_wi_data.wi_stopping = 1;
437 spin_unlock(&cfs_wi_data.wi_glock);
439 /* nobody should contend on this list */
440 list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
441 sched->ws_stopping = 1;
442 wake_up_all(&sched->ws_waitq);
445 list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
446 spin_lock(&cfs_wi_data.wi_glock);
448 while (sched->ws_nthreads != 0) {
449 spin_unlock(&cfs_wi_data.wi_glock);
450 set_current_state(TASK_UNINTERRUPTIBLE);
451 schedule_timeout(cfs_time_seconds(1) / 20);
452 spin_lock(&cfs_wi_data.wi_glock);
454 spin_unlock(&cfs_wi_data.wi_glock);
456 while (!list_empty(&cfs_wi_data.wi_scheds)) {
457 sched = list_entry(cfs_wi_data.wi_scheds.next,
458 struct cfs_wi_sched, ws_list);
459 list_del(&sched->ws_list);
460 LIBCFS_FREE(sched, sizeof(*sched));
463 cfs_wi_data.wi_stopping = 0;
464 cfs_wi_data.wi_init = 0;