2 * Copyright (C) 2011 Red Hat, Inc.
4 * This file is released under the GPL.
6 #include "dm-block-manager.h"
7 #include "dm-persistent-data-internal.h"
8 #include "../dm-bufio.h"
10 #include <linux/crc32c.h>
11 #include <linux/module.h>
12 #include <linux/slab.h>
13 #include <linux/rwsem.h>
14 #include <linux/device-mapper.h>
15 #include <linux/stacktrace.h>
17 #define DM_MSG_PREFIX "block manager"
19 /*----------------------------------------------------------------*/
22 * This is a read/write semaphore with a couple of differences.
24 * i) There is a restriction on the number of concurrent read locks that
25 * may be held at once. This is just an implementation detail.
27 * ii) Recursive locking attempts are detected and return EINVAL. A stack
28 * trace is also emitted for the previous lock aquisition.
30 * iii) Priority is given to write locks.
35 typedef unsigned long stack_entries[MAX_STACK];
40 struct list_head waiters;
41 struct task_struct *holders[MAX_HOLDERS];
43 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
44 struct stack_trace traces[MAX_HOLDERS];
45 stack_entries entries[MAX_HOLDERS];
50 struct list_head list;
51 struct task_struct *task;
55 static unsigned __find_holder(struct block_lock *lock,
56 struct task_struct *task)
60 for (i = 0; i < MAX_HOLDERS; i++)
61 if (lock->holders[i] == task)
64 BUG_ON(i == MAX_HOLDERS);
68 /* call this *after* you increment lock->count */
69 static void __add_holder(struct block_lock *lock, struct task_struct *task)
71 unsigned h = __find_holder(lock, NULL);
72 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
73 struct stack_trace *t;
76 get_task_struct(task);
77 lock->holders[h] = task;
79 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
82 t->max_entries = MAX_STACK;
83 t->entries = lock->entries[h];
89 /* call this *before* you decrement lock->count */
90 static void __del_holder(struct block_lock *lock, struct task_struct *task)
92 unsigned h = __find_holder(lock, task);
93 lock->holders[h] = NULL;
94 put_task_struct(task);
97 static int __check_holder(struct block_lock *lock)
100 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
101 static struct stack_trace t;
102 static stack_entries entries;
105 for (i = 0; i < MAX_HOLDERS; i++) {
106 if (lock->holders[i] == current) {
107 DMERR("recursive lock detected in pool metadata");
108 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
109 DMERR("previously held here:");
110 print_stack_trace(lock->traces + i, 4);
112 DMERR("subsequent aquisition attempted here:");
114 t.max_entries = MAX_STACK;
117 save_stack_trace(&t);
118 print_stack_trace(&t, 4);
127 static void __wait(struct waiter *w)
130 set_task_state(current, TASK_UNINTERRUPTIBLE);
138 set_task_state(current, TASK_RUNNING);
141 static void __wake_waiter(struct waiter *w)
143 struct task_struct *task;
149 wake_up_process(task);
153 * We either wake a few readers or a single writer.
155 static void __wake_many(struct block_lock *lock)
157 struct waiter *w, *tmp;
159 BUG_ON(lock->count < 0);
160 list_for_each_entry_safe(w, tmp, &lock->waiters, list) {
161 if (lock->count >= MAX_HOLDERS)
164 if (w->wants_write) {
166 return; /* still read locked */
169 __add_holder(lock, w->task);
175 __add_holder(lock, w->task);
180 static void bl_init(struct block_lock *lock)
184 spin_lock_init(&lock->lock);
186 INIT_LIST_HEAD(&lock->waiters);
187 for (i = 0; i < MAX_HOLDERS; i++)
188 lock->holders[i] = NULL;
191 static int __available_for_read(struct block_lock *lock)
193 return lock->count >= 0 &&
194 lock->count < MAX_HOLDERS &&
195 list_empty(&lock->waiters);
198 static int bl_down_read(struct block_lock *lock)
203 spin_lock(&lock->lock);
204 r = __check_holder(lock);
206 spin_unlock(&lock->lock);
210 if (__available_for_read(lock)) {
212 __add_holder(lock, current);
213 spin_unlock(&lock->lock);
217 get_task_struct(current);
221 list_add_tail(&w.list, &lock->waiters);
222 spin_unlock(&lock->lock);
225 put_task_struct(current);
229 static int bl_down_read_nonblock(struct block_lock *lock)
233 spin_lock(&lock->lock);
234 r = __check_holder(lock);
238 if (__available_for_read(lock)) {
240 __add_holder(lock, current);
246 spin_unlock(&lock->lock);
250 static void bl_up_read(struct block_lock *lock)
252 spin_lock(&lock->lock);
253 BUG_ON(lock->count <= 0);
254 __del_holder(lock, current);
256 if (!list_empty(&lock->waiters))
258 spin_unlock(&lock->lock);
261 static int bl_down_write(struct block_lock *lock)
266 spin_lock(&lock->lock);
267 r = __check_holder(lock);
269 spin_unlock(&lock->lock);
273 if (lock->count == 0 && list_empty(&lock->waiters)) {
275 __add_holder(lock, current);
276 spin_unlock(&lock->lock);
280 get_task_struct(current);
285 * Writers given priority. We know there's only one mutator in the
286 * system, so ignoring the ordering reversal.
288 list_add(&w.list, &lock->waiters);
289 spin_unlock(&lock->lock);
292 put_task_struct(current);
297 static void bl_up_write(struct block_lock *lock)
299 spin_lock(&lock->lock);
300 __del_holder(lock, current);
302 if (!list_empty(&lock->waiters))
304 spin_unlock(&lock->lock);
307 static void report_recursive_bug(dm_block_t b, int r)
310 DMERR("recursive acquisition of block %llu requested.",
311 (unsigned long long) b);
314 /*----------------------------------------------------------------*/
317 * Block manager is currently implemented using dm-bufio. struct
318 * dm_block_manager and struct dm_block map directly onto a couple of
319 * structs in the bufio interface. I want to retain the freedom to move
320 * away from bufio in the future. So these structs are just cast within
321 * this .c file, rather than making it through to the public interface.
323 static struct dm_buffer *to_buffer(struct dm_block *b)
325 return (struct dm_buffer *) b;
328 static struct dm_bufio_client *to_bufio(struct dm_block_manager *bm)
330 return (struct dm_bufio_client *) bm;
333 dm_block_t dm_block_location(struct dm_block *b)
335 return dm_bufio_get_block_number(to_buffer(b));
337 EXPORT_SYMBOL_GPL(dm_block_location);
339 void *dm_block_data(struct dm_block *b)
341 return dm_bufio_get_block_data(to_buffer(b));
343 EXPORT_SYMBOL_GPL(dm_block_data);
346 struct dm_block_validator *validator;
347 struct block_lock lock;
351 static void dm_block_manager_alloc_callback(struct dm_buffer *buf)
353 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
354 aux->validator = NULL;
358 static void dm_block_manager_write_callback(struct dm_buffer *buf)
360 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
361 if (aux->validator) {
362 aux->validator->prepare_for_write(aux->validator, (struct dm_block *) buf,
363 dm_bufio_get_block_size(dm_bufio_get_client(buf)));
367 /*----------------------------------------------------------------
369 *--------------------------------------------------------------*/
370 struct dm_block_manager *dm_block_manager_create(struct block_device *bdev,
373 unsigned max_held_per_thread)
375 return (struct dm_block_manager *)
376 dm_bufio_client_create(bdev, block_size, max_held_per_thread,
377 sizeof(struct buffer_aux),
378 dm_block_manager_alloc_callback,
379 dm_block_manager_write_callback);
381 EXPORT_SYMBOL_GPL(dm_block_manager_create);
383 void dm_block_manager_destroy(struct dm_block_manager *bm)
385 return dm_bufio_client_destroy(to_bufio(bm));
387 EXPORT_SYMBOL_GPL(dm_block_manager_destroy);
389 unsigned dm_bm_block_size(struct dm_block_manager *bm)
391 return dm_bufio_get_block_size(to_bufio(bm));
393 EXPORT_SYMBOL_GPL(dm_bm_block_size);
395 dm_block_t dm_bm_nr_blocks(struct dm_block_manager *bm)
397 return dm_bufio_get_device_size(to_bufio(bm));
400 static int dm_bm_validate_buffer(struct dm_block_manager *bm,
401 struct dm_buffer *buf,
402 struct buffer_aux *aux,
403 struct dm_block_validator *v)
405 if (unlikely(!aux->validator)) {
409 r = v->check(v, (struct dm_block *) buf, dm_bufio_get_block_size(to_bufio(bm)));
414 if (unlikely(aux->validator != v)) {
415 DMERR("validator mismatch (old=%s vs new=%s) for block %llu",
416 aux->validator->name, v ? v->name : "NULL",
418 dm_bufio_get_block_number(buf));
425 int dm_bm_read_lock(struct dm_block_manager *bm, dm_block_t b,
426 struct dm_block_validator *v,
427 struct dm_block **result)
429 struct buffer_aux *aux;
433 p = dm_bufio_read(to_bufio(bm), b, (struct dm_buffer **) result);
434 if (unlikely(IS_ERR(p)))
437 aux = dm_bufio_get_aux_data(to_buffer(*result));
438 r = bl_down_read(&aux->lock);
440 dm_bufio_release(to_buffer(*result));
441 report_recursive_bug(b, r);
445 aux->write_locked = 0;
447 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
449 bl_up_read(&aux->lock);
450 dm_bufio_release(to_buffer(*result));
456 EXPORT_SYMBOL_GPL(dm_bm_read_lock);
458 int dm_bm_write_lock(struct dm_block_manager *bm,
459 dm_block_t b, struct dm_block_validator *v,
460 struct dm_block **result)
462 struct buffer_aux *aux;
466 p = dm_bufio_read(to_bufio(bm), b, (struct dm_buffer **) result);
467 if (unlikely(IS_ERR(p)))
470 aux = dm_bufio_get_aux_data(to_buffer(*result));
471 r = bl_down_write(&aux->lock);
473 dm_bufio_release(to_buffer(*result));
474 report_recursive_bug(b, r);
478 aux->write_locked = 1;
480 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
482 bl_up_write(&aux->lock);
483 dm_bufio_release(to_buffer(*result));
489 EXPORT_SYMBOL_GPL(dm_bm_write_lock);
491 int dm_bm_read_try_lock(struct dm_block_manager *bm,
492 dm_block_t b, struct dm_block_validator *v,
493 struct dm_block **result)
495 struct buffer_aux *aux;
499 p = dm_bufio_get(to_bufio(bm), b, (struct dm_buffer **) result);
500 if (unlikely(IS_ERR(p)))
505 aux = dm_bufio_get_aux_data(to_buffer(*result));
506 r = bl_down_read_nonblock(&aux->lock);
508 dm_bufio_release(to_buffer(*result));
509 report_recursive_bug(b, r);
512 aux->write_locked = 0;
514 r = dm_bm_validate_buffer(bm, to_buffer(*result), aux, v);
516 bl_up_read(&aux->lock);
517 dm_bufio_release(to_buffer(*result));
524 int dm_bm_write_lock_zero(struct dm_block_manager *bm,
525 dm_block_t b, struct dm_block_validator *v,
526 struct dm_block **result)
529 struct buffer_aux *aux;
532 p = dm_bufio_new(to_bufio(bm), b, (struct dm_buffer **) result);
533 if (unlikely(IS_ERR(p)))
536 memset(p, 0, dm_bm_block_size(bm));
538 aux = dm_bufio_get_aux_data(to_buffer(*result));
539 r = bl_down_write(&aux->lock);
541 dm_bufio_release(to_buffer(*result));
545 aux->write_locked = 1;
551 int dm_bm_unlock(struct dm_block *b)
553 struct buffer_aux *aux;
554 aux = dm_bufio_get_aux_data(to_buffer(b));
556 if (aux->write_locked) {
557 dm_bufio_mark_buffer_dirty(to_buffer(b));
558 bl_up_write(&aux->lock);
560 bl_up_read(&aux->lock);
562 dm_bufio_release(to_buffer(b));
566 EXPORT_SYMBOL_GPL(dm_bm_unlock);
568 int dm_bm_unlock_move(struct dm_block *b, dm_block_t n)
570 struct buffer_aux *aux;
572 aux = dm_bufio_get_aux_data(to_buffer(b));
574 if (aux->write_locked) {
575 dm_bufio_mark_buffer_dirty(to_buffer(b));
576 bl_up_write(&aux->lock);
578 bl_up_read(&aux->lock);
580 dm_bufio_release_move(to_buffer(b), n);
584 int dm_bm_flush_and_unlock(struct dm_block_manager *bm,
585 struct dm_block *superblock)
589 r = dm_bufio_write_dirty_buffers(to_bufio(bm));
592 r = dm_bufio_issue_flush(to_bufio(bm));
596 dm_bm_unlock(superblock);
598 r = dm_bufio_write_dirty_buffers(to_bufio(bm));
601 r = dm_bufio_issue_flush(to_bufio(bm));
608 u32 dm_bm_checksum(const void *data, size_t len, u32 init_xor)
610 return crc32c(~(u32) 0, data, len) ^ init_xor;
612 EXPORT_SYMBOL_GPL(dm_bm_checksum);
614 /*----------------------------------------------------------------*/
616 MODULE_LICENSE("GPL");
617 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
618 MODULE_DESCRIPTION("Immutable metadata library for dm");
620 /*----------------------------------------------------------------*/