DEFINE_TRACE(block_split);
+/*
+ * Test patch to inline a certain number of bi_io_vec's inside the bio
+ * itself, to shrink a bio data allocation from two mempool calls to one
+ */
+#define BIO_INLINE_VECS 4
+
static mempool_t *bio_split_pool __read_mostly;
/*
* kzalloc() for the exact number of vecs right away.
*/
if (!bs)
- bvl = kzalloc(nr * sizeof(struct bio_vec), gfp_mask);
+ bvl = kmalloc(nr * sizeof(struct bio_vec), gfp_mask);
/*
* see comment near bvec_array define!
}
}
- if (bvl)
- memset(bvl, 0, bvec_nr_vecs(*idx) * sizeof(struct bio_vec));
-
return bvl;
}
{
void *p;
- if (bio->bi_io_vec)
+ if (bio_has_allocated_vec(bio))
bvec_free_bs(bs, bio->bi_io_vec, BIO_POOL_IDX(bio));
if (bio_integrity(bio))
static void bio_kmalloc_destructor(struct bio *bio)
{
- kfree(bio->bi_io_vec);
+ if (bio_has_allocated_vec(bio))
+ kfree(bio->bi_io_vec);
kfree(bio);
}
if (likely(nr_iovecs)) {
unsigned long uninitialized_var(idx);
- bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx, bs);
+ if (nr_iovecs <= BIO_INLINE_VECS) {
+ idx = 0;
+ bvl = bio->bi_inline_vecs;
+ nr_iovecs = BIO_INLINE_VECS;
+ } else {
+ bvl = bvec_alloc_bs(gfp_mask, nr_iovecs, &idx,
+ bs);
+ nr_iovecs = bvec_nr_vecs(idx);
+ }
if (unlikely(!bvl)) {
if (bs)
mempool_free(bio, bs->bio_pool);
goto out;
}
bio->bi_flags |= idx << BIO_POOL_OFFSET;
- bio->bi_max_vecs = bvec_nr_vecs(idx);
+ bio->bi_max_vecs = nr_iovecs;
}
bio->bi_io_vec = bvl;
}
*/
struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
{
+ unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
struct bio_set *bs;
bs = kzalloc(sizeof(*bs), GFP_KERNEL);
bs->front_pad = front_pad;
- bs->bio_slab = bio_find_or_create_slab(front_pad);
+ bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
if (!bs->bio_slab) {
kfree(bs);
return NULL;