From 2b17cb931723a707ed26ddf1bea053be8e0c7964 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Wed, 5 Oct 2011 11:44:04 +1100 Subject: [PATCH] dio: use a slab cache for struct dio A direct slab call is slightly faster than kmalloc and can be better cached per CPU. It also avoids rounding to the next kmalloc slab. In addition this enforces cache line alignment for struct dio to avoid any false sharing. Signed-off-by: Andi Kleen Acked-by: Jeff Moyer Cc: Jens Axboe Cc: Christoph Hellwig Signed-off-by: Andrew Morton --- fs/direct-io.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/fs/direct-io.c b/fs/direct-io.c index 2d07a69431ab..2443c48c43c7 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -140,7 +140,9 @@ struct dio { * wish that they not be zeroed. */ struct page *pages[DIO_PAGES]; /* page buffer */ -}; +} ____cacheline_aligned_in_smp; + +static struct kmem_cache *dio_cache __read_mostly; static void __inode_dio_wait(struct inode *inode) { @@ -330,7 +332,7 @@ static void dio_bio_end_aio(struct bio *bio, int error) if (remaining == 0) { dio_complete(dio, dio->iocb->ki_pos, 0, true); - kfree(dio); + kmem_cache_free(dio_cache, dio); } } @@ -1174,7 +1176,7 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode, if (ret2 == 0) { ret = dio_complete(dio, offset, ret, false); - kfree(dio); + kmem_cache_free(dio_cache, dio); } else BUG_ON(ret != -EIOCBQUEUED); @@ -1250,7 +1252,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, if (rw == READ && end == offset) return 0; - dio = kmalloc(sizeof(*dio), GFP_KERNEL); + dio = kmem_cache_alloc(dio_cache, GFP_KERNEL); retval = -ENOMEM; if (!dio) goto out; @@ -1274,7 +1276,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, end - 1); if (retval) { mutex_unlock(&inode->i_mutex); - kfree(dio); + kmem_cache_free(dio_cache, dio); goto out; } } @@ -1302,3 +1304,10 @@ out: return retval; } EXPORT_SYMBOL(__blockdev_direct_IO); + +static __init int dio_init(void) +{ + dio_cache = KMEM_CACHE(dio, SLAB_PANIC); + return 0; +} +module_init(dio_init) -- 2.39.5