diff options
| -rw-r--r-- | fs/aio.c | 155 |
1 files changed, 74 insertions, 81 deletions
@@ -58,18 +58,6 @@ struct aio_ring { }; /* 128 bytes + ring size */ #define AIO_RING_PAGES 8 -struct aio_ring_info { - unsigned long mmap_base; - unsigned long mmap_size; - - struct page **ring_pages; - struct mutex ring_lock; - long nr_pages; - - unsigned nr, tail; - - struct page *internal_pages[AIO_RING_PAGES]; -}; struct kioctx { atomic_t users; @@ -90,14 +78,30 @@ struct kioctx { * This is what userspace passed to io_setup(), it's not used for * anything but counting against the global max_reqs quota. * - * The real limit is ring->nr - 1, which will be larger (see + * The real limit is nr_events - 1, which will be larger (see * aio_setup_ring()) */ unsigned max_reqs; - struct aio_ring_info ring_info; + /* Size of ringbuffer, in units of struct io_event */ + unsigned nr_events; - spinlock_t completion_lock; + unsigned long mmap_base; + unsigned long mmap_size; + + struct page **ring_pages; + long nr_pages; + + struct { + struct mutex ring_lock; + } ____cacheline_aligned; + + struct { + unsigned tail; + spinlock_t completion_lock; + } ____cacheline_aligned; + + struct page *internal_pages[AIO_RING_PAGES]; struct rcu_head rcu_head; struct work_struct rcu_work; @@ -129,26 +133,21 @@ __initcall(aio_setup); static void aio_free_ring(struct kioctx *ctx) { - struct aio_ring_info *info = &ctx->ring_info; long i; - for (i=0; i<info->nr_pages; i++) - put_page(info->ring_pages[i]); + for (i = 0; i < ctx->nr_pages; i++) + put_page(ctx->ring_pages[i]); - if (info->mmap_size) { - vm_munmap(info->mmap_base, info->mmap_size); - } + if (ctx->mmap_size) + vm_munmap(ctx->mmap_base, ctx->mmap_size); - if (info->ring_pages && info->ring_pages != info->internal_pages) - kfree(info->ring_pages); - info->ring_pages = NULL; - info->nr = 0; + if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) + kfree(ctx->ring_pages); } static int aio_setup_ring(struct kioctx *ctx) { struct aio_ring *ring; - struct aio_ring_info *info = &ctx->ring_info; unsigned nr_events = ctx->max_reqs; struct mm_struct *mm = current->mm; unsigned long size, populate; @@ -166,45 +165,44 @@ static int aio_setup_ring(struct kioctx *ctx) nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) / sizeof(struct io_event); - info->nr = 0; - info->ring_pages = info->internal_pages; + ctx->nr_events = 0; + ctx->ring_pages = ctx->internal_pages; if (nr_pages > AIO_RING_PAGES) { - info->ring_pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); - if (!info->ring_pages) + ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *), + GFP_KERNEL); + if (!ctx->ring_pages) return -ENOMEM; } - info->mmap_size = nr_pages * PAGE_SIZE; - pr_debug("attempting mmap of %lu bytes\n", info->mmap_size); + ctx->mmap_size = nr_pages * PAGE_SIZE; + pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); down_write(&mm->mmap_sem); - info->mmap_base = do_mmap_pgoff(NULL, 0, info->mmap_size, - PROT_READ|PROT_WRITE, - MAP_ANONYMOUS|MAP_PRIVATE, 0, - &populate); - if (IS_ERR((void *)info->mmap_base)) { + ctx->mmap_base = do_mmap_pgoff(NULL, 0, ctx->mmap_size, + PROT_READ|PROT_WRITE, + MAP_ANONYMOUS|MAP_PRIVATE, 0, &populate); + if (IS_ERR((void *)ctx->mmap_base)) { up_write(&mm->mmap_sem); - info->mmap_size = 0; + ctx->mmap_size = 0; aio_free_ring(ctx); return -EAGAIN; } - pr_debug("mmap address: 0x%08lx\n", info->mmap_base); - info->nr_pages = get_user_pages(current, mm, info->mmap_base, nr_pages, - 1, 0, info->ring_pages, NULL); + pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); + ctx->nr_pages = get_user_pages(current, mm, ctx->mmap_base, nr_pages, + 1, 0, ctx->ring_pages, NULL); up_write(&mm->mmap_sem); - if (unlikely(info->nr_pages != nr_pages)) { + if (unlikely(ctx->nr_pages != nr_pages)) { aio_free_ring(ctx); return -EAGAIN; } if (populate) - mm_populate(info->mmap_base, populate); + mm_populate(ctx->mmap_base, populate); - ctx->user_id = info->mmap_base; + ctx->user_id = ctx->mmap_base; + ctx->nr_events = nr_events; /* trusted copy */ - info->nr = nr_events; /* trusted copy */ - - ring = kmap_atomic(info->ring_pages[0]); + ring = kmap_atomic(ctx->ring_pages[0]); ring->nr = nr_events; /* user copy */ ring->id = ctx->user_id; ring->head = ring->tail = 0; @@ -213,7 +211,7 @@ static int aio_setup_ring(struct kioctx *ctx) ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; ring->header_length = sizeof(struct aio_ring); kunmap_atomic(ring); - flush_dcache_page(info->ring_pages[0]); + flush_dcache_page(ctx->ring_pages[0]); return 0; } @@ -284,7 +282,6 @@ static void free_ioctx_rcu(struct rcu_head *head) */ static void free_ioctx(struct kioctx *ctx) { - struct aio_ring_info *info = &ctx->ring_info; struct aio_ring *ring; struct io_event res; struct kiocb *req; @@ -302,18 +299,18 @@ static void free_ioctx(struct kioctx *ctx) spin_unlock_irq(&ctx->ctx_lock); - ring = kmap_atomic(info->ring_pages[0]); + ring = kmap_atomic(ctx->ring_pages[0]); head = ring->head; kunmap_atomic(ring); while (atomic_read(&ctx->reqs_active) > 0) { - wait_event(ctx->wait, head != info->tail); + wait_event(ctx->wait, head != ctx->tail); - avail = (head <= info->tail ? info->tail : info->nr) - head; + avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head; atomic_sub(avail, &ctx->reqs_active); head += avail; - head %= info->nr; + head %= ctx->nr_events; } WARN_ON(atomic_read(&ctx->reqs_active) < 0); @@ -372,7 +369,7 @@ static struct kioctx *ioctx_alloc(unsigned nr_events) atomic_set(&ctx->dead, 0); spin_lock_init(&ctx->ctx_lock); spin_lock_init(&ctx->completion_lock); - mutex_init(&ctx->ring_info.ring_lock); + mutex_init(&ctx->ring_lock); init_waitqueue_head(&ctx->wait); INIT_LIST_HEAD(&ctx->active_reqs); @@ -396,7 +393,7 @@ static struct kioctx *ioctx_alloc(unsigned nr_events) spin_unlock(&mm->ioctx_lock); pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", - ctx, ctx->user_id, mm, ctx->ring_info.nr); + ctx, ctx->user_id, mm, ctx->nr_events); return ctx; out_cleanup: @@ -491,7 +488,7 @@ void exit_aio(struct mm_struct *mm) * just set it to 0; aio_free_ring() is the only * place that uses ->mmap_size, so it's safe. */ - ctx->ring_info.mmap_size = 0; + ctx->mmap_size = 0; if (!atomic_xchg(&ctx->dead, 1)) { hlist_del_rcu(&ctx->list); @@ -514,10 +511,10 @@ static inline struct kiocb *aio_get_req(struct kioctx *ctx) { struct kiocb *req; - if (atomic_read(&ctx->reqs_active) >= ctx->ring_info.nr) + if (atomic_read(&ctx->reqs_active) >= ctx->nr_events) return NULL; - if (atomic_inc_return(&ctx->reqs_active) > ctx->ring_info.nr - 1) + if (atomic_inc_return(&ctx->reqs_active) > ctx->nr_events - 1) goto out_put; req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO); @@ -578,7 +575,6 @@ static struct kioctx *lookup_ioctx(unsigned long ctx_id) void aio_complete(struct kiocb *iocb, long res, long res2) { struct kioctx *ctx = iocb->ki_ctx; - struct aio_ring_info *info; struct aio_ring *ring; struct io_event *ev_page, *event; unsigned long flags; @@ -599,8 +595,6 @@ void aio_complete(struct kiocb *iocb, long res, long res2) return; } - info = &ctx->ring_info; - /* * Take rcu_read_lock() in case the kioctx is being destroyed, as we * need to issue a wakeup after decrementing reqs_active. @@ -633,13 +627,13 @@ void aio_complete(struct kiocb *iocb, long res, long res2) */ spin_lock_irqsave(&ctx->completion_lock, flags); - tail = info->tail; + tail = ctx->tail; pos = tail + AIO_EVENTS_OFFSET; - if (++tail >= info->nr) + if (++tail >= ctx->nr_events) tail = 0; - ev_page = kmap_atomic(info->ring_pages[pos / AIO_EVENTS_PER_PAGE]); + ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); event = ev_page + pos % AIO_EVENTS_PER_PAGE; event->obj = (u64)(unsigned long)iocb->ki_obj.user; @@ -648,7 +642,7 @@ void aio_complete(struct kiocb *iocb, long res, long res2) event->res2 = res2; kunmap_atomic(ev_page); - flush_dcache_page(info->ring_pages[pos / AIO_EVENTS_PER_PAGE]); + flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n", ctx, tail, iocb, iocb->ki_obj.user, iocb->ki_user_data, @@ -659,12 +653,12 @@ void aio_complete(struct kiocb *iocb, long res, long res2) */ smp_wmb(); /* make event visible before updating tail */ - info->tail = tail; + ctx->tail = tail; - ring = kmap_atomic(info->ring_pages[0]); + ring = kmap_atomic(ctx->ring_pages[0]); ring->tail = tail; kunmap_atomic(ring); - flush_dcache_page(info->ring_pages[0]); + flush_dcache_page(ctx->ring_pages[0]); spin_unlock_irqrestore(&ctx->completion_lock, flags); @@ -704,21 +698,20 @@ EXPORT_SYMBOL(aio_complete); static long aio_read_events_ring(struct kioctx *ctx, struct io_event __user *event, long nr) { - struct aio_ring_info *info = &ctx->ring_info; struct aio_ring *ring; unsigned head, pos; long ret = 0; int copy_ret; - mutex_lock(&info->ring_lock); + mutex_lock(&ctx->ring_lock); - ring = kmap_atomic(info->ring_pages[0]); + ring = kmap_atomic(ctx->ring_pages[0]); head = ring->head; kunmap_atomic(ring); - pr_debug("h%u t%u m%u\n", head, info->tail, info->nr); + pr_debug("h%u t%u m%u\n", head, ctx->tail, ctx->nr_events); - if (head == info->tail) + if (head == ctx->tail) goto out; while (ret < nr) { @@ -726,8 +719,8 @@ static long aio_read_events_ring(struct kioctx *ctx, struct io_event *ev; struct page *page; - avail = (head <= info->tail ? info->tail : info->nr) - head; - if (head == info->tail) + avail = (head <= ctx->tail ? ctx->tail : ctx->nr_events) - head; + if (head == ctx->tail) break; avail = min(avail, nr - ret); @@ -735,7 +728,7 @@ static long aio_read_events_ring(struct kioctx *ctx, ((head + AIO_EVENTS_OFFSET) % AIO_EVENTS_PER_PAGE)); pos = head + AIO_EVENTS_OFFSET; - page = info->ring_pages[pos / AIO_EVENTS_PER_PAGE]; + page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]; pos %= AIO_EVENTS_PER_PAGE; ev = kmap(page); @@ -750,19 +743,19 @@ static long aio_read_events_ring(struct kioctx *ctx, ret += avail; head += avail; - head %= info->nr; + head %= ctx->nr_events; } - ring = kmap_atomic(info->ring_pages[0]); + ring = kmap_atomic(ctx->ring_pages[0]); ring->head = head; kunmap_atomic(ring); - flush_dcache_page(info->ring_pages[0]); + flush_dcache_page(ctx->ring_pages[0]); - pr_debug("%li h%u t%u\n", ret, head, info->tail); + pr_debug("%li h%u t%u\n", ret, head, ctx->tail); atomic_sub(ret, &ctx->reqs_active); out: - mutex_unlock(&info->ring_lock); + mutex_unlock(&ctx->ring_lock); return ret; } |