2 * Functions related to mapping data to requests
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/blkdev.h>
8 #include <linux/scatterlist.h>
9 #include <linux/slab.h>
10 #include <scsi/sg.h> /* for struct sg_iovec */
14 int blk_rq_append_bio(struct request_queue *q, struct request *rq,
18 blk_rq_bio_prep(q, rq, bio);
19 else if (!ll_back_merge_fn(q, rq, bio))
22 rq->biotail->bi_next = bio;
25 rq->__data_len += bio->bi_size;
30 static int __blk_rq_unmap_user(struct bio *bio)
35 if (bio_flagged(bio, BIO_USER_MAPPED))
38 ret = bio_uncopy_user(bio);
44 static int __blk_rq_map_user(struct request_queue *q, struct request *rq,
45 struct rq_map_data *map_data, void __user *ubuf,
46 unsigned int len, gfp_t gfp_mask)
49 struct bio *bio, *orig_bio;
52 reading = rq_data_dir(rq) == READ;
55 * if alignment requirement is satisfied, map in user pages for
56 * direct dma. else, set up kernel bounce buffers
58 uaddr = (unsigned long) ubuf;
59 if (blk_rq_aligned(q, uaddr, len) && !map_data)
60 bio = bio_map_user(q, NULL, uaddr, len, reading, gfp_mask);
62 bio = bio_copy_user(q, map_data, uaddr, len, reading, gfp_mask);
67 if (map_data && map_data->null_mapped)
68 bio->bi_flags |= (1 << BIO_NULL_MAPPED);
71 blk_queue_bounce(q, &bio);
74 * We link the bounce buffer in and could have to traverse it
75 * later so we have to get a ref to prevent it from being freed
79 ret = blk_rq_append_bio(q, rq, bio);
83 /* if it was boucned we must call the end io function */
85 __blk_rq_unmap_user(orig_bio);
91 * blk_rq_map_user - map user data to a request, for REQ_TYPE_BLOCK_PC usage
92 * @q: request queue where request should be inserted
93 * @rq: request structure to fill
94 * @map_data: pointer to the rq_map_data holding pages (if necessary)
95 * @ubuf: the user buffer
96 * @len: length of user data
97 * @gfp_mask: memory allocation flags
100 * Data will be mapped directly for zero copy I/O, if possible. Otherwise
101 * a kernel bounce buffer is used.
103 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
104 * still in process context.
106 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
107 * before being submitted to the device, as pages mapped may be out of
108 * reach. It's the callers responsibility to make sure this happens. The
109 * original bio must be passed back in to blk_rq_unmap_user() for proper
112 int blk_rq_map_user(struct request_queue *q, struct request *rq,
113 struct rq_map_data *map_data, void __user *ubuf,
114 unsigned long len, gfp_t gfp_mask)
116 unsigned long bytes_read = 0;
117 struct bio *bio = NULL;
120 if (len > (queue_max_hw_sectors(q) << 9))
125 if (!ubuf && (!map_data || !map_data->null_mapped))
128 while (bytes_read != len) {
129 unsigned long map_len, end, start;
131 map_len = min_t(unsigned long, len - bytes_read, BIO_MAX_SIZE);
132 end = ((unsigned long)ubuf + map_len + PAGE_SIZE - 1)
134 start = (unsigned long)ubuf >> PAGE_SHIFT;
137 * A bad offset could cause us to require BIO_MAX_PAGES + 1
138 * pages. If this happens we just lower the requested
139 * mapping len by a page so that we can fit
141 if (end - start > BIO_MAX_PAGES)
142 map_len -= PAGE_SIZE;
144 ret = __blk_rq_map_user(q, rq, map_data, ubuf, map_len,
154 map_data->offset += ret;
157 if (!bio_flagged(bio, BIO_USER_MAPPED))
158 rq->cmd_flags |= REQ_COPY_USER;
163 blk_rq_unmap_user(bio);
167 EXPORT_SYMBOL(blk_rq_map_user);
170 * blk_rq_map_user_iov - map user data to a request, for REQ_TYPE_BLOCK_PC usage
171 * @q: request queue where request should be inserted
172 * @rq: request to map data to
173 * @map_data: pointer to the rq_map_data holding pages (if necessary)
174 * @iov: pointer to the iovec
175 * @iov_count: number of elements in the iovec
176 * @len: I/O byte count
177 * @gfp_mask: memory allocation flags
180 * Data will be mapped directly for zero copy I/O, if possible. Otherwise
181 * a kernel bounce buffer is used.
183 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
184 * still in process context.
186 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
187 * before being submitted to the device, as pages mapped may be out of
188 * reach. It's the callers responsibility to make sure this happens. The
189 * original bio must be passed back in to blk_rq_unmap_user() for proper
192 int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
193 struct rq_map_data *map_data, struct sg_iovec *iov,
194 int iov_count, unsigned int len, gfp_t gfp_mask)
197 int i, read = rq_data_dir(rq) == READ;
200 if (!iov || iov_count <= 0)
203 for (i = 0; i < iov_count; i++) {
204 unsigned long uaddr = (unsigned long)iov[i].iov_base;
209 if (uaddr & queue_dma_alignment(q)) {
215 if (unaligned || (q->dma_pad_mask & len) || map_data)
216 bio = bio_copy_user_iov(q, map_data, iov, iov_count, read,
219 bio = bio_map_user_iov(q, NULL, iov, iov_count, read, gfp_mask);
224 if (bio->bi_size != len) {
226 * Grab an extra reference to this bio, as bio_unmap_user()
227 * expects to be able to drop it twice as it happens on the
228 * normal IO completion path
232 __blk_rq_unmap_user(bio);
236 if (!bio_flagged(bio, BIO_USER_MAPPED))
237 rq->cmd_flags |= REQ_COPY_USER;
239 blk_queue_bounce(q, &bio);
241 blk_rq_bio_prep(q, rq, bio);
245 EXPORT_SYMBOL(blk_rq_map_user_iov);
248 * blk_rq_unmap_user - unmap a request with user data
249 * @bio: start of bio list
252 * Unmap a rq previously mapped by blk_rq_map_user(). The caller must
253 * supply the original rq->bio from the blk_rq_map_user() return, since
254 * the I/O completion may have changed rq->bio.
256 int blk_rq_unmap_user(struct bio *bio)
258 struct bio *mapped_bio;
263 if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
264 mapped_bio = bio->bi_private;
266 ret2 = __blk_rq_unmap_user(mapped_bio);
277 EXPORT_SYMBOL(blk_rq_unmap_user);
279 struct blk_kern_sg_work {
280 atomic_t bios_inflight;
281 struct sg_table sg_table;
282 struct scatterlist *src_sgl;
285 static void blk_free_kern_sg_work(struct blk_kern_sg_work *bw)
287 sg_free_table(&bw->sg_table);
292 static void blk_bio_map_kern_endio(struct bio *bio, int err)
294 struct blk_kern_sg_work *bw = bio->bi_private;
297 /* Decrement the bios in processing and, if zero, free */
298 BUG_ON(atomic_read(&bw->bios_inflight) <= 0);
299 if (atomic_dec_and_test(&bw->bios_inflight)) {
300 if ((bio_data_dir(bio) == READ) && (err == 0)) {
303 local_irq_save(flags); /* to protect KMs */
304 sg_copy(bw->src_sgl, bw->sg_table.sgl, 0, 0,
305 KM_BIO_DST_IRQ, KM_BIO_SRC_IRQ);
306 local_irq_restore(flags);
308 blk_free_kern_sg_work(bw);
316 static int blk_rq_copy_kern_sg(struct request *rq, struct scatterlist *sgl,
317 int nents, struct blk_kern_sg_work **pbw,
318 gfp_t gfp, gfp_t page_gfp)
321 struct scatterlist *sg;
322 struct scatterlist *new_sgl;
324 size_t len = 0, to_copy;
325 struct blk_kern_sg_work *bw;
327 bw = kzalloc(sizeof(*bw), gfp);
333 for_each_sg(sgl, sg, nents, i)
337 new_sgl_nents = PFN_UP(len);
339 res = sg_alloc_table(&bw->sg_table, new_sgl_nents, gfp);
343 new_sgl = bw->sg_table.sgl;
345 for_each_sg(new_sgl, sg, new_sgl_nents, i) {
348 pg = alloc_page(page_gfp);
350 goto err_free_new_sgl;
352 sg_assign_page(sg, pg);
353 sg->length = min_t(size_t, PAGE_SIZE, len);
358 if (rq_data_dir(rq) == WRITE) {
360 * We need to limit amount of copied data to to_copy, because
361 * sgl might have the last element in sgl not marked as last in
364 sg_copy(new_sgl, sgl, 0, to_copy,
370 * REQ_COPY_USER name is misleading. It should be something like
371 * REQ_HAS_TAIL_SPACE_FOR_PADDING.
373 rq->cmd_flags |= REQ_COPY_USER;
379 for_each_sg(new_sgl, sg, new_sgl_nents, i) {
380 struct page *pg = sg_page(sg);
385 sg_free_table(&bw->sg_table);
393 static int __blk_rq_map_kern_sg(struct request *rq, struct scatterlist *sgl,
394 int nents, struct blk_kern_sg_work *bw, gfp_t gfp)
397 struct request_queue *q = rq->q;
398 int rw = rq_data_dir(rq);
402 struct scatterlist *sg, *prev_sg = NULL;
403 struct bio *bio = NULL, *hbio = NULL, *tbio = NULL;
406 if (unlikely((sgl == NULL) || (sgl->length == 0) || (nents <= 0))) {
413 * Let's keep each bio allocation inside a single page to decrease
414 * probability of failure.
416 max_nr_vecs = min_t(size_t,
417 ((PAGE_SIZE - sizeof(struct bio)) / sizeof(struct bio_vec)),
423 for_each_sg(sgl, sg, nents, i) {
424 struct page *page = sg_page(sg);
425 void *page_addr = page_address(page);
426 size_t len = sg->length, l;
427 size_t offset = sg->offset;
433 * Each segment must be aligned on DMA boundary and
434 * not on stack. The last one may have unaligned
435 * length as long as the total length is aligned to
436 * DMA padding alignment.
442 if (((sg->offset | l) & queue_dma_alignment(q)) ||
443 (page_addr && object_is_on_stack(page_addr + sg->offset))) {
453 bio = bio_kmalloc(gfp, max_nr_vecs);
460 bio->bi_rw |= 1 << BIO_RW;
463 bio->bi_private = bw;
464 bio->bi_end_io = blk_bio_map_kern_endio;
469 tbio = tbio->bi_next = bio;
472 bytes = min_t(size_t, len, PAGE_SIZE - offset);
474 rc = bio_add_pc_page(q, bio, page, bytes, offset);
476 if (unlikely(need_new_bio || (rc < 0))) {
490 need_new_bio = false;
493 page = nth_page(page, 1);
502 /* Total length must be aligned on DMA padding alignment */
503 if ((tot_len & q->dma_pad_mask) &&
504 !(rq->cmd_flags & REQ_COPY_USER)) {
510 atomic_set(&bw->bios_inflight, bios);
512 while (hbio != NULL) {
514 hbio = hbio->bi_next;
517 blk_queue_bounce(q, &bio);
519 res = blk_rq_append_bio(q, rq, bio);
520 if (unlikely(res != 0)) {
523 /* We can have one or more bios bounced */
533 while (hbio != NULL) {
535 hbio = hbio->bi_next;
541 blk_rq_unmap_kern_sg(rq, res);
546 * blk_rq_map_kern_sg - map kernel data to a request, for REQ_TYPE_BLOCK_PC
547 * @rq: request to fill
549 * @nents: number of elements in @sgl
550 * @gfp: memory allocation flags
553 * Data will be mapped directly if possible. Otherwise a bounce
554 * buffer will be used.
556 int blk_rq_map_kern_sg(struct request *rq, struct scatterlist *sgl,
557 int nents, gfp_t gfp)
561 res = __blk_rq_map_kern_sg(rq, sgl, nents, NULL, gfp);
562 if (unlikely(res != 0)) {
563 struct blk_kern_sg_work *bw = NULL;
565 res = blk_rq_copy_kern_sg(rq, sgl, nents, &bw,
566 gfp, rq->q->bounce_gfp | gfp);
567 if (unlikely(res != 0))
570 res = __blk_rq_map_kern_sg(rq, bw->sg_table.sgl,
571 bw->sg_table.nents, bw, gfp);
573 blk_free_kern_sg_work(bw);
583 EXPORT_SYMBOL(blk_rq_map_kern_sg);
586 * blk_rq_unmap_kern_sg - unmap a request with kernel sg
587 * @rq: request to unmap
588 * @err: non-zero error code
591 * Unmap a rq previously mapped by blk_rq_map_kern_sg(). Must be called
592 * only in case of an error!
594 void blk_rq_unmap_kern_sg(struct request *rq, int err)
596 struct bio *bio = rq->bio;
601 b->bi_end_io(b, err);
607 EXPORT_SYMBOL(blk_rq_unmap_kern_sg);
610 * blk_rq_map_kern - map kernel data to a request, for REQ_TYPE_BLOCK_PC usage
611 * @q: request queue where request should be inserted
612 * @rq: request to fill
613 * @kbuf: the kernel buffer
614 * @len: length of user data
615 * @gfp_mask: memory allocation flags
618 * Data will be mapped directly if possible. Otherwise a bounce
619 * buffer is used. Can be called multple times to append multple
622 int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
623 unsigned int len, gfp_t gfp_mask)
625 int reading = rq_data_dir(rq) == READ;
626 unsigned long addr = (unsigned long) kbuf;
631 if (len > (queue_max_hw_sectors(q) << 9))
636 do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf);
638 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
640 bio = bio_map_kern(q, kbuf, len, gfp_mask);
645 if (rq_data_dir(rq) == WRITE)
646 bio->bi_rw |= REQ_WRITE;
649 rq->cmd_flags |= REQ_COPY_USER;
651 ret = blk_rq_append_bio(q, rq, bio);
653 /* request is too big */
658 blk_queue_bounce(q, &rq->bio);
662 EXPORT_SYMBOL(blk_rq_map_kern);