X-Git-Url: https://git.karo-electronics.de/?a=blobdiff_plain;f=fs%2Faio.c;h=0e11e31dbb77147078b79eaad3fc97c70b79321d;hb=656da9da3745abcbbbdca598745d04c6de2c8843;hp=a82214d2e46d81c56a8e3858e9af427758e8cd49;hpb=25ee7e3832951cf5896b194f6cd929a44863f419;p=mv-sheeva.git diff --git a/fs/aio.c b/fs/aio.c index a82214d2e46..0e11e31dbb7 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -40,9 +41,6 @@ #define dprintk(x...) do { ; } while (0) #endif -static long aio_run = 0; /* for testing only */ -static long aio_wakeups = 0; /* for testing only */ - /*------ sysctl variables----*/ atomic_t aio_nr = ATOMIC_INIT(0); /* current system wide number of aio requests */ unsigned aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ @@ -61,6 +59,7 @@ static DEFINE_SPINLOCK(fput_lock); static LIST_HEAD(fput_head); static void aio_kick_handler(void *); +static void aio_queue_work(struct kioctx *); /* aio_setup * Creates the slab caches used by the aio routines, panic on @@ -405,7 +404,6 @@ static struct kiocb fastcall *__aio_get_req(struct kioctx *ctx) req->ki_ctx = ctx; req->ki_cancel = NULL; req->ki_retry = NULL; - req->ki_obj.user = NULL; req->ki_dtor = NULL; req->private = NULL; INIT_LIST_HEAD(&req->ki_run_list); @@ -451,11 +449,6 @@ static inline void really_put_req(struct kioctx *ctx, struct kiocb *req) { if (req->ki_dtor) req->ki_dtor(req); - req->ki_ctx = NULL; - req->ki_filp = NULL; - req->ki_obj.user = NULL; - req->ki_dtor = NULL; - req->private = NULL; kmem_cache_free(kiocb_cachep, req); ctx->reqs_active--; @@ -507,7 +500,7 @@ static int __aio_put_req(struct kioctx *ctx, struct kiocb *req) /* Must be done under the lock to serialise against cancellation. * Call this aio_fput as it duplicates fput via the fput_work. */ - if (unlikely(atomic_dec_and_test(&req->ki_filp->f_count))) { + if (unlikely(rcuref_dec_and_test(&req->ki_filp->f_count))) { get_ioctx(ctx); spin_lock(&fput_lock); list_add(&req->ki_list, &fput_head); @@ -554,6 +547,25 @@ struct kioctx *lookup_ioctx(unsigned long ctx_id) return ioctx; } +static int lock_kiocb_action(void *param) +{ + schedule(); + return 0; +} + +static inline void lock_kiocb(struct kiocb *iocb) +{ + wait_on_bit_lock(&iocb->ki_flags, KIF_LOCKED, lock_kiocb_action, + TASK_UNINTERRUPTIBLE); +} + +static inline void unlock_kiocb(struct kiocb *iocb) +{ + kiocbClearLocked(iocb); + smp_mb__after_clear_bit(); + wake_up_bit(&iocb->ki_flags, KIF_LOCKED); +} + /* * use_mm * Makes the calling kernel thread take on the specified @@ -575,6 +587,10 @@ static void use_mm(struct mm_struct *mm) atomic_inc(&mm->mm_count); tsk->mm = mm; tsk->active_mm = mm; + /* + * Note that on UML this *requires* PF_BORROWED_MM to be set, otherwise + * it won't work. Update it accordingly if you change it here + */ activate_mm(active_mm, mm); task_unlock(tsk); @@ -623,7 +639,6 @@ static inline int __queue_kicked_iocb(struct kiocb *iocb) if (list_empty(&iocb->ki_run_list)) { list_add_tail(&iocb->ki_run_list, &ctx->run_list); - iocb->ki_queued++; return 1; } return 0; @@ -664,10 +679,8 @@ static ssize_t aio_run_iocb(struct kiocb *iocb) } if (!(iocb->ki_retried & 0xff)) { - pr_debug("%ld retry: %d of %d (kick %ld, Q %ld run %ld, wake %ld)\n", - iocb->ki_retried, - iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes, - iocb->ki_kicked, iocb->ki_queued, aio_run, aio_wakeups); + pr_debug("%ld retry: %d of %d\n", iocb->ki_retried, + iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes); } if (!(retry = iocb->ki_retry)) { @@ -759,6 +772,14 @@ out: * has already been kicked */ if (kiocbIsKicked(iocb)) { __queue_kicked_iocb(iocb); + + /* + * __queue_kicked_iocb will always return 1 here, because + * iocb->ki_run_list is empty at this point so it should + * be safe to unconditionally queue the context into the + * work queue. + */ + aio_queue_work(ctx); } } return ret; @@ -774,7 +795,6 @@ out: static int __aio_run_iocbs(struct kioctx *ctx) { struct kiocb *iocb; - int count = 0; LIST_HEAD(run_list); list_splice_init(&ctx->run_list, &run_list); @@ -786,12 +806,12 @@ static int __aio_run_iocbs(struct kioctx *ctx) * Hold an extra reference while retrying i/o. */ iocb->ki_users++; /* grab extra reference */ + lock_kiocb(iocb); aio_run_iocb(iocb); + unlock_kiocb(iocb); if (__aio_put_req(ctx, iocb)) /* drop extra ref */ put_ioctx(ctx); - count++; } - aio_run++; if (!list_empty(&ctx->run_list)) return 1; return 0; @@ -890,10 +910,8 @@ static void queue_kicked_iocb(struct kiocb *iocb) spin_lock_irqsave(&ctx->ctx_lock, flags); run = __queue_kicked_iocb(iocb); spin_unlock_irqrestore(&ctx->ctx_lock, flags); - if (run) { + if (run) aio_queue_work(ctx); - aio_wakeups++; - } } /* @@ -913,7 +931,6 @@ void fastcall kick_iocb(struct kiocb *iocb) return; } - iocb->ki_kicked++; /* If its already kicked we shouldn't queue it again */ if (!kiocbTryKick(iocb)) { queue_kicked_iocb(iocb); @@ -984,7 +1001,8 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) tail = info->tail; event = aio_ring_event(info, tail, KM_IRQ0); - tail = (tail + 1) % info->nr; + if (++tail >= info->nr) + tail = 0; event->obj = (u64)(unsigned long)iocb->ki_obj.user; event->data = iocb->ki_user_data; @@ -1008,10 +1026,8 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) pr_debug("added to ring %p at [%lu]\n", iocb, tail); - pr_debug("%ld retries: %d of %d (kicked %ld, Q %ld run %ld wake %ld)\n", - iocb->ki_retried, - iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes, - iocb->ki_kicked, iocb->ki_queued, aio_run, aio_wakeups); + pr_debug("%ld retries: %d of %d\n", iocb->ki_retried, + iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes); put_rq: /* everything turned out well, dispose of the aiocb. */ ret = __aio_put_req(ctx, iocb); @@ -1119,7 +1135,6 @@ static int read_events(struct kioctx *ctx, int i = 0; struct io_event ent; struct aio_timeout to; - int event_loop = 0; /* testing only */ int retry = 0; /* needed to zero any padding within an entry (there shouldn't be @@ -1186,7 +1201,6 @@ retry: if (to.timed_out) /* Only check after read evt */ break; schedule(); - event_loop++; if (signal_pending(tsk)) { ret = -EINTR; break; @@ -1214,9 +1228,6 @@ retry: if (timeout) clear_timeout(&to); out: - pr_debug("event loop executed %d times\n", event_loop); - pr_debug("aio_run %ld\n", aio_run); - pr_debug("aio_wakeups %ld\n", aio_wakeups); return i ? i : ret; } @@ -1515,8 +1526,7 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, } req->ki_filp = file; - iocb->aio_key = req->ki_key; - ret = put_user(iocb->aio_key, &user_iocb->aio_key); + ret = put_user(req->ki_key, &user_iocb->aio_key); if (unlikely(ret)) { dprintk("EFAULT: aio_key\n"); goto out_put_req; @@ -1531,13 +1541,7 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, req->ki_opcode = iocb->aio_lio_opcode; init_waitqueue_func_entry(&req->ki_wait, aio_wake_function); INIT_LIST_HEAD(&req->ki_wait.task_list); - req->ki_run_list.next = req->ki_run_list.prev = NULL; - req->ki_retry = NULL; req->ki_retried = 0; - req->ki_kicked = 0; - req->ki_queued = 0; - aio_run = 0; - aio_wakeups = 0; ret = aio_setup_iocb(req); @@ -1545,10 +1549,13 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, goto out_put_req; spin_lock_irq(&ctx->ctx_lock); - list_add_tail(&req->ki_run_list, &ctx->run_list); - /* drain the run list */ - while (__aio_run_iocbs(ctx)) - ; + aio_run_iocb(req); + unlock_kiocb(req); + if (!list_empty(&ctx->run_list)) { + /* drain the run list */ + while (__aio_run_iocbs(ctx)) + ; + } spin_unlock_irq(&ctx->ctx_lock); aio_put_req(req); /* drop extra ref to req */ return 0; @@ -1675,6 +1682,7 @@ asmlinkage long sys_io_cancel(aio_context_t ctx_id, struct iocb __user *iocb, if (NULL != cancel) { struct io_event tmp; pr_debug("calling cancel\n"); + lock_kiocb(kiocb); memset(&tmp, 0, sizeof(tmp)); tmp.obj = (u64)(unsigned long)kiocb->ki_obj.user; tmp.data = kiocb->ki_user_data; @@ -1686,8 +1694,9 @@ asmlinkage long sys_io_cancel(aio_context_t ctx_id, struct iocb __user *iocb, if (copy_to_user(result, &tmp, sizeof(tmp))) ret = -EFAULT; } + unlock_kiocb(kiocb); } else - printk(KERN_DEBUG "iocb has no cancel operation\n"); + ret = -EINVAL; put_ioctx(ctx);