- Most significant change this cycle is request-based DM now supports
stacking ontop of blk-mq devices. This blk-mq support changes the model request-based DM uses for cloning a request to relying on calling blk_get_request() directly from the underlying blk-mq device. Early consumer of this code is Intel's emerging NVMe hardware; thanks to Keith Busch for working on, and pushing for, these changes. - A few other small fixes and cleanups across other DM targets. -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQEcBAABAgAGBQJU3NRnAAoJEMUj8QotnQNavG0H/3yogMcHvKg9H+w0WmUQdwhN w99Wj3nkquAw2sm9yahKlAMBNY53iu/LHmC6/PaTpJetgdH7y1foTrRa0qjyeB2D DgNr8mOzxSxzX6CX9V8JMwqzky9XoG2IOt/7FeQQOpMqp4T1M2zgvbZtpl0lK/f3 lNaNBFpl+47NbGssD/WbtfI4Yy3hX0u406yGmQN5DxRyGTWD2AFqpA76g2mp8vrp wmw259gPr4oLhj3pDc0GkuiVn59ZR2Zp+2gs0jD5uKlDL84VP/nE+WNB+ny1Mnmt cOg8Q+W6/OosL66MKBHNsF0QS6DXNo5UvsN9fHGa5IUJw7Tsa11ZEPKHZGEbQw4= =RiN2 -----END PGP SIGNATURE----- Merge tag 'dm-3.20-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm Pull device mapper changes from Mike Snitzer: - The most significant change this cycle is request-based DM now supports stacking ontop of blk-mq devices. This blk-mq support changes the model request-based DM uses for cloning a request to relying on calling blk_get_request() directly from the underlying blk-mq device. An early consumer of this code is Intel's emerging NVMe hardware; thanks to Keith Busch for working on, and pushing for, these changes. - A few other small fixes and cleanups across other DM targets. * tag 'dm-3.20-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: dm: inherit QUEUE_FLAG_SG_GAPS flags from underlying queues dm snapshot: remove unnecessary NULL checks before vfree() calls dm mpath: simplify failure path of dm_multipath_init() dm thin metadata: remove unused dm_pool_get_data_block_size() dm ioctl: fix stale comment above dm_get_inactive_table() dm crypt: update url in CONFIG_DM_CRYPT help text dm bufio: fix time comparison to use time_after_eq() dm: use time_in_range() and time_after() dm raid: fix a couple integer overflows dm table: train hybrid target type detection to select blk-mq if appropriate dm: allocate requests in target when stacking on blk-mq devices dm: prepare for allocating blk-mq clone requests in target dm: submit stacked requests in irq enabled context dm: split request structure out from dm_rq_target_io structure dm: remove exports for request-based interfaces without external callers
This commit is contained in:
Linus Torvalds
commit
802ea9d864
17 changed files with 415 additions and 197 deletions
|
|
@ -231,9 +231,8 @@ config DM_CRYPT
|
|||
transparently encrypts the data on it. You'll need to activate
|
||||
the ciphers you're going to use in the cryptoapi configuration.
|
||||
|
||||
Information on how to use dm-crypt can be found on
|
||||
|
||||
<http://www.saout.de/misc/dm-crypt/>
|
||||
For further information on dm-crypt and userspace tools see:
|
||||
<http://code.google.com/p/cryptsetup/wiki/DMCrypt>
|
||||
|
||||
To compile this code as a module, choose M here: the module will
|
||||
be called dm-crypt.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include <linux/device-mapper.h>
|
||||
#include <linux/dm-io.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/shrinker.h>
|
||||
#include <linux/module.h>
|
||||
|
|
@ -1739,7 +1740,7 @@ static unsigned get_max_age_hz(void)
|
|||
|
||||
static bool older_than(struct dm_buffer *b, unsigned long age_hz)
|
||||
{
|
||||
return (jiffies - b->last_accessed) >= age_hz;
|
||||
return time_after_eq(jiffies, b->last_accessed + age_hz);
|
||||
}
|
||||
|
||||
static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
|
||||
#include <linux/dm-io.h>
|
||||
#include <linux/dm-kcopyd.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/mempool.h>
|
||||
#include <linux/module.h>
|
||||
|
|
@ -1562,8 +1563,8 @@ static void process_bio(struct cache *cache, struct prealloc *structs,
|
|||
|
||||
static int need_commit_due_to_time(struct cache *cache)
|
||||
{
|
||||
return jiffies < cache->last_commit_jiffies ||
|
||||
jiffies > cache->last_commit_jiffies + COMMIT_PERIOD;
|
||||
return !time_in_range(jiffies, cache->last_commit_jiffies,
|
||||
cache->last_commit_jiffies + COMMIT_PERIOD);
|
||||
}
|
||||
|
||||
static int commit_if_needed(struct cache *cache)
|
||||
|
|
|
|||
|
|
@ -639,8 +639,8 @@ static int check_name(const char *name)
|
|||
|
||||
/*
|
||||
* On successful return, the caller must not attempt to acquire
|
||||
* _hash_lock without first calling dm_table_put, because dm_table_destroy
|
||||
* waits for this dm_table_put and could be called under this lock.
|
||||
* _hash_lock without first calling dm_put_live_table, because dm_table_destroy
|
||||
* waits for this dm_put_live_table and could be called under this lock.
|
||||
*/
|
||||
static struct dm_table *dm_get_inactive_table(struct mapped_device *md, int *srcu_idx)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#include <linux/bio.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/dm-dirty-log.h>
|
||||
#include <linux/device-mapper.h>
|
||||
#include <linux/dm-log-userspace.h>
|
||||
|
|
@ -829,7 +830,7 @@ static int userspace_is_remote_recovering(struct dm_dirty_log *log,
|
|||
int r;
|
||||
uint64_t region64 = region;
|
||||
struct log_c *lc = log->context;
|
||||
static unsigned long long limit;
|
||||
static unsigned long limit;
|
||||
struct {
|
||||
int64_t is_recovering;
|
||||
uint64_t in_sync_hint;
|
||||
|
|
@ -845,7 +846,7 @@ static int userspace_is_remote_recovering(struct dm_dirty_log *log,
|
|||
*/
|
||||
if (region < lc->in_sync_hint)
|
||||
return 0;
|
||||
else if (jiffies < limit)
|
||||
else if (time_after(limit, jiffies))
|
||||
return 1;
|
||||
|
||||
limit = jiffies + (HZ / 4);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include "dm-path-selector.h"
|
||||
#include "dm-uevent.h"
|
||||
|
||||
#include <linux/blkdev.h>
|
||||
#include <linux/ctype.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/mempool.h>
|
||||
|
|
@ -378,18 +379,18 @@ static int __must_push_back(struct multipath *m)
|
|||
/*
|
||||
* Map cloned requests
|
||||
*/
|
||||
static int multipath_map(struct dm_target *ti, struct request *clone,
|
||||
union map_info *map_context)
|
||||
static int __multipath_map(struct dm_target *ti, struct request *clone,
|
||||
union map_info *map_context,
|
||||
struct request *rq, struct request **__clone)
|
||||
{
|
||||
struct multipath *m = (struct multipath *) ti->private;
|
||||
int r = DM_MAPIO_REQUEUE;
|
||||
size_t nr_bytes = blk_rq_bytes(clone);
|
||||
unsigned long flags;
|
||||
size_t nr_bytes = clone ? blk_rq_bytes(clone) : blk_rq_bytes(rq);
|
||||
struct pgpath *pgpath;
|
||||
struct block_device *bdev;
|
||||
struct dm_mpath_io *mpio;
|
||||
|
||||
spin_lock_irqsave(&m->lock, flags);
|
||||
spin_lock_irq(&m->lock);
|
||||
|
||||
/* Do we need to select a new pgpath? */
|
||||
if (!m->current_pgpath ||
|
||||
|
|
@ -411,25 +412,61 @@ static int multipath_map(struct dm_target *ti, struct request *clone,
|
|||
/* ENOMEM, requeue */
|
||||
goto out_unlock;
|
||||
|
||||
bdev = pgpath->path.dev->bdev;
|
||||
clone->q = bdev_get_queue(bdev);
|
||||
clone->rq_disk = bdev->bd_disk;
|
||||
clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
|
||||
mpio = map_context->ptr;
|
||||
mpio->pgpath = pgpath;
|
||||
mpio->nr_bytes = nr_bytes;
|
||||
|
||||
bdev = pgpath->path.dev->bdev;
|
||||
|
||||
spin_unlock_irq(&m->lock);
|
||||
|
||||
if (clone) {
|
||||
/* Old request-based interface: allocated clone is passed in */
|
||||
clone->q = bdev_get_queue(bdev);
|
||||
clone->rq_disk = bdev->bd_disk;
|
||||
clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
|
||||
} else {
|
||||
/* blk-mq request-based interface */
|
||||
*__clone = blk_get_request(bdev_get_queue(bdev),
|
||||
rq_data_dir(rq), GFP_KERNEL);
|
||||
if (IS_ERR(*__clone))
|
||||
/* ENOMEM, requeue */
|
||||
return r;
|
||||
(*__clone)->bio = (*__clone)->biotail = NULL;
|
||||
(*__clone)->rq_disk = bdev->bd_disk;
|
||||
(*__clone)->cmd_flags |= REQ_FAILFAST_TRANSPORT;
|
||||
}
|
||||
|
||||
if (pgpath->pg->ps.type->start_io)
|
||||
pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
|
||||
&pgpath->path,
|
||||
nr_bytes);
|
||||
r = DM_MAPIO_REMAPPED;
|
||||
return DM_MAPIO_REMAPPED;
|
||||
|
||||
out_unlock:
|
||||
spin_unlock_irqrestore(&m->lock, flags);
|
||||
spin_unlock_irq(&m->lock);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
static int multipath_map(struct dm_target *ti, struct request *clone,
|
||||
union map_info *map_context)
|
||||
{
|
||||
return __multipath_map(ti, clone, map_context, NULL, NULL);
|
||||
}
|
||||
|
||||
static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
|
||||
union map_info *map_context,
|
||||
struct request **clone)
|
||||
{
|
||||
return __multipath_map(ti, NULL, map_context, rq, clone);
|
||||
}
|
||||
|
||||
static void multipath_release_clone(struct request *clone)
|
||||
{
|
||||
blk_put_request(clone);
|
||||
}
|
||||
|
||||
/*
|
||||
* If we run out of usable paths, should we queue I/O or error it?
|
||||
*/
|
||||
|
|
@ -1666,11 +1703,13 @@ out:
|
|||
*---------------------------------------------------------------*/
|
||||
static struct target_type multipath_target = {
|
||||
.name = "multipath",
|
||||
.version = {1, 7, 0},
|
||||
.version = {1, 8, 0},
|
||||
.module = THIS_MODULE,
|
||||
.ctr = multipath_ctr,
|
||||
.dtr = multipath_dtr,
|
||||
.map_rq = multipath_map,
|
||||
.clone_and_map_rq = multipath_clone_and_map,
|
||||
.release_clone_rq = multipath_release_clone,
|
||||
.rq_end_io = multipath_end_io,
|
||||
.presuspend = multipath_presuspend,
|
||||
.postsuspend = multipath_postsuspend,
|
||||
|
|
@ -1694,16 +1733,15 @@ static int __init dm_multipath_init(void)
|
|||
r = dm_register_target(&multipath_target);
|
||||
if (r < 0) {
|
||||
DMERR("register failed %d", r);
|
||||
kmem_cache_destroy(_mpio_cache);
|
||||
return -EINVAL;
|
||||
r = -EINVAL;
|
||||
goto bad_register_target;
|
||||
}
|
||||
|
||||
kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
|
||||
if (!kmultipathd) {
|
||||
DMERR("failed to create workqueue kmpathd");
|
||||
dm_unregister_target(&multipath_target);
|
||||
kmem_cache_destroy(_mpio_cache);
|
||||
return -ENOMEM;
|
||||
r = -ENOMEM;
|
||||
goto bad_alloc_kmultipathd;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1716,16 +1754,23 @@ static int __init dm_multipath_init(void)
|
|||
WQ_MEM_RECLAIM);
|
||||
if (!kmpath_handlerd) {
|
||||
DMERR("failed to create workqueue kmpath_handlerd");
|
||||
destroy_workqueue(kmultipathd);
|
||||
dm_unregister_target(&multipath_target);
|
||||
kmem_cache_destroy(_mpio_cache);
|
||||
return -ENOMEM;
|
||||
r = -ENOMEM;
|
||||
goto bad_alloc_kmpath_handlerd;
|
||||
}
|
||||
|
||||
DMINFO("version %u.%u.%u loaded",
|
||||
multipath_target.version[0], multipath_target.version[1],
|
||||
multipath_target.version[2]);
|
||||
|
||||
return 0;
|
||||
|
||||
bad_alloc_kmpath_handlerd:
|
||||
destroy_workqueue(kmultipathd);
|
||||
bad_alloc_kmultipathd:
|
||||
dm_unregister_target(&multipath_target);
|
||||
bad_register_target:
|
||||
kmem_cache_destroy(_mpio_cache);
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1237,7 +1237,7 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|||
argv++;
|
||||
|
||||
/* Skip over RAID params for now and find out # of devices */
|
||||
if (num_raid_params + 1 > argc) {
|
||||
if (num_raid_params >= argc) {
|
||||
ti->error = "Arguments do not agree with counts given";
|
||||
return -EINVAL;
|
||||
}
|
||||
|
|
@ -1248,6 +1248,12 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
argc -= num_raid_params + 1; /* +1: we already have num_raid_devs */
|
||||
if (argc != (num_raid_devs * 2)) {
|
||||
ti->error = "Supplied RAID devices does not match the count given";
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rs = context_alloc(ti, rt, (unsigned)num_raid_devs);
|
||||
if (IS_ERR(rs))
|
||||
return PTR_ERR(rs);
|
||||
|
|
@ -1256,16 +1262,8 @@ static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
|
|||
if (ret)
|
||||
goto bad;
|
||||
|
||||
ret = -EINVAL;
|
||||
|
||||
argc -= num_raid_params + 1; /* +1: we already have num_raid_devs */
|
||||
argv += num_raid_params + 1;
|
||||
|
||||
if (argc != (num_raid_devs * 2)) {
|
||||
ti->error = "Supplied RAID devices does not match the count given";
|
||||
goto bad;
|
||||
}
|
||||
|
||||
ret = dev_parms(rs, argv);
|
||||
if (ret)
|
||||
goto bad;
|
||||
|
|
|
|||
|
|
@ -200,16 +200,11 @@ err_area:
|
|||
|
||||
static void free_area(struct pstore *ps)
|
||||
{
|
||||
if (ps->area)
|
||||
vfree(ps->area);
|
||||
vfree(ps->area);
|
||||
ps->area = NULL;
|
||||
|
||||
if (ps->zero_area)
|
||||
vfree(ps->zero_area);
|
||||
vfree(ps->zero_area);
|
||||
ps->zero_area = NULL;
|
||||
|
||||
if (ps->header_area)
|
||||
vfree(ps->header_area);
|
||||
vfree(ps->header_area);
|
||||
ps->header_area = NULL;
|
||||
}
|
||||
|
||||
|
|
@ -605,8 +600,7 @@ static void persistent_dtr(struct dm_exception_store *store)
|
|||
free_area(ps);
|
||||
|
||||
/* Allocated in persistent_read_metadata */
|
||||
if (ps->callbacks)
|
||||
vfree(ps->callbacks);
|
||||
vfree(ps->callbacks);
|
||||
|
||||
kfree(ps);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -827,10 +827,11 @@ static int dm_table_set_type(struct dm_table *t)
|
|||
{
|
||||
unsigned i;
|
||||
unsigned bio_based = 0, request_based = 0, hybrid = 0;
|
||||
bool use_blk_mq = false;
|
||||
struct dm_target *tgt;
|
||||
struct dm_dev_internal *dd;
|
||||
struct list_head *devices;
|
||||
unsigned live_md_type;
|
||||
unsigned live_md_type = dm_get_md_type(t->md);
|
||||
|
||||
for (i = 0; i < t->num_targets; i++) {
|
||||
tgt = t->targets + i;
|
||||
|
|
@ -854,8 +855,8 @@ static int dm_table_set_type(struct dm_table *t)
|
|||
* Determine the type from the live device.
|
||||
* Default to bio-based if device is new.
|
||||
*/
|
||||
live_md_type = dm_get_md_type(t->md);
|
||||
if (live_md_type == DM_TYPE_REQUEST_BASED)
|
||||
if (live_md_type == DM_TYPE_REQUEST_BASED ||
|
||||
live_md_type == DM_TYPE_MQ_REQUEST_BASED)
|
||||
request_based = 1;
|
||||
else
|
||||
bio_based = 1;
|
||||
|
|
@ -869,16 +870,6 @@ static int dm_table_set_type(struct dm_table *t)
|
|||
|
||||
BUG_ON(!request_based); /* No targets in this table */
|
||||
|
||||
/* Non-request-stackable devices can't be used for request-based dm */
|
||||
devices = dm_table_get_devices(t);
|
||||
list_for_each_entry(dd, devices, list) {
|
||||
if (!blk_queue_stackable(bdev_get_queue(dd->dm_dev->bdev))) {
|
||||
DMWARN("table load rejected: including"
|
||||
" non-request-stackable devices");
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Request-based dm supports only tables that have a single target now.
|
||||
* To support multiple targets, request splitting support is needed,
|
||||
|
|
@ -890,7 +881,37 @@ static int dm_table_set_type(struct dm_table *t)
|
|||
return -EINVAL;
|
||||
}
|
||||
|
||||
t->type = DM_TYPE_REQUEST_BASED;
|
||||
/* Non-request-stackable devices can't be used for request-based dm */
|
||||
devices = dm_table_get_devices(t);
|
||||
list_for_each_entry(dd, devices, list) {
|
||||
struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
|
||||
|
||||
if (!blk_queue_stackable(q)) {
|
||||
DMERR("table load rejected: including"
|
||||
" non-request-stackable devices");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (q->mq_ops)
|
||||
use_blk_mq = true;
|
||||
}
|
||||
|
||||
if (use_blk_mq) {
|
||||
/* verify _all_ devices in the table are blk-mq devices */
|
||||
list_for_each_entry(dd, devices, list)
|
||||
if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) {
|
||||
DMERR("table load rejected: not all devices"
|
||||
" are blk-mq request-stackable");
|
||||
return -EINVAL;
|
||||
}
|
||||
t->type = DM_TYPE_MQ_REQUEST_BASED;
|
||||
|
||||
} else if (hybrid && list_empty(devices) && live_md_type != DM_TYPE_NONE) {
|
||||
/* inherit live MD type */
|
||||
t->type = live_md_type;
|
||||
|
||||
} else
|
||||
t->type = DM_TYPE_REQUEST_BASED;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -907,7 +928,15 @@ struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
|
|||
|
||||
bool dm_table_request_based(struct dm_table *t)
|
||||
{
|
||||
return dm_table_get_type(t) == DM_TYPE_REQUEST_BASED;
|
||||
unsigned table_type = dm_table_get_type(t);
|
||||
|
||||
return (table_type == DM_TYPE_REQUEST_BASED ||
|
||||
table_type == DM_TYPE_MQ_REQUEST_BASED);
|
||||
}
|
||||
|
||||
bool dm_table_mq_request_based(struct dm_table *t)
|
||||
{
|
||||
return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED;
|
||||
}
|
||||
|
||||
static int dm_table_alloc_md_mempools(struct dm_table *t)
|
||||
|
|
@ -1360,6 +1389,14 @@ static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
|
|||
return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
|
||||
}
|
||||
|
||||
static int queue_supports_sg_gaps(struct dm_target *ti, struct dm_dev *dev,
|
||||
sector_t start, sector_t len, void *data)
|
||||
{
|
||||
struct request_queue *q = bdev_get_queue(dev->bdev);
|
||||
|
||||
return q && !test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags);
|
||||
}
|
||||
|
||||
static bool dm_table_all_devices_attribute(struct dm_table *t,
|
||||
iterate_devices_callout_fn func)
|
||||
{
|
||||
|
|
@ -1480,6 +1517,11 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
|
|||
else
|
||||
queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
|
||||
|
||||
if (dm_table_all_devices_attribute(t, queue_supports_sg_gaps))
|
||||
queue_flag_clear_unlocked(QUEUE_FLAG_SG_GAPS, q);
|
||||
else
|
||||
queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, q);
|
||||
|
||||
dm_table_set_integrity(t);
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -137,13 +137,26 @@ static int io_err_map_rq(struct dm_target *ti, struct request *clone,
|
|||
return -EIO;
|
||||
}
|
||||
|
||||
static int io_err_clone_and_map_rq(struct dm_target *ti, struct request *rq,
|
||||
union map_info *map_context,
|
||||
struct request **clone)
|
||||
{
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
static void io_err_release_clone_rq(struct request *clone)
|
||||
{
|
||||
}
|
||||
|
||||
static struct target_type error_target = {
|
||||
.name = "error",
|
||||
.version = {1, 2, 0},
|
||||
.version = {1, 3, 0},
|
||||
.ctr = io_err_ctr,
|
||||
.dtr = io_err_dtr,
|
||||
.map = io_err_map,
|
||||
.map_rq = io_err_map_rq,
|
||||
.clone_and_map_rq = io_err_clone_and_map_rq,
|
||||
.release_clone_rq = io_err_release_clone_rq,
|
||||
};
|
||||
|
||||
int __init dm_target_init(void)
|
||||
|
|
|
|||
|
|
@ -1635,15 +1635,6 @@ int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,
|
|||
return r;
|
||||
}
|
||||
|
||||
int dm_pool_get_data_block_size(struct dm_pool_metadata *pmd, sector_t *result)
|
||||
{
|
||||
down_read(&pmd->root_lock);
|
||||
*result = pmd->data_block_size;
|
||||
up_read(&pmd->root_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result)
|
||||
{
|
||||
int r = -EINVAL;
|
||||
|
|
|
|||
|
|
@ -182,8 +182,6 @@ int dm_pool_get_free_metadata_block_count(struct dm_pool_metadata *pmd,
|
|||
int dm_pool_get_metadata_dev_size(struct dm_pool_metadata *pmd,
|
||||
dm_block_t *result);
|
||||
|
||||
int dm_pool_get_data_block_size(struct dm_pool_metadata *pmd, sector_t *result);
|
||||
|
||||
int dm_pool_get_data_dev_size(struct dm_pool_metadata *pmd, dm_block_t *result);
|
||||
|
||||
int dm_pool_block_is_used(struct dm_pool_metadata *pmd, dm_block_t b, bool *result);
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
#include <linux/device-mapper.h>
|
||||
#include <linux/dm-io.h>
|
||||
#include <linux/dm-kcopyd.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/log2.h>
|
||||
#include <linux/list.h>
|
||||
#include <linux/rculist.h>
|
||||
|
|
@ -1700,8 +1701,8 @@ static void process_cell_fail(struct thin_c *tc, struct dm_bio_prison_cell *cell
|
|||
*/
|
||||
static int need_commit_due_to_time(struct pool *pool)
|
||||
{
|
||||
return jiffies < pool->last_commit_jiffies ||
|
||||
jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
|
||||
return !time_in_range(jiffies, pool->last_commit_jiffies,
|
||||
pool->last_commit_jiffies + COMMIT_PERIOD);
|
||||
}
|
||||
|
||||
#define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
|
||||
|
|
|
|||
345
drivers/md/dm.c
345
drivers/md/dm.c
|
|
@ -20,6 +20,7 @@
|
|||
#include <linux/hdreg.h>
|
||||
#include <linux/delay.h>
|
||||
#include <linux/wait.h>
|
||||
#include <linux/kthread.h>
|
||||
|
||||
#include <trace/events/block.h>
|
||||
|
||||
|
|
@ -78,7 +79,8 @@ struct dm_io {
|
|||
struct dm_rq_target_io {
|
||||
struct mapped_device *md;
|
||||
struct dm_target *ti;
|
||||
struct request *orig, clone;
|
||||
struct request *orig, *clone;
|
||||
struct kthread_work work;
|
||||
int error;
|
||||
union map_info info;
|
||||
};
|
||||
|
|
@ -179,6 +181,7 @@ struct mapped_device {
|
|||
* io objects are allocated from here.
|
||||
*/
|
||||
mempool_t *io_pool;
|
||||
mempool_t *rq_pool;
|
||||
|
||||
struct bio_set *bs;
|
||||
|
||||
|
|
@ -210,6 +213,9 @@ struct mapped_device {
|
|||
unsigned internal_suspend_count;
|
||||
|
||||
struct dm_stats stats;
|
||||
|
||||
struct kthread_worker kworker;
|
||||
struct task_struct *kworker_task;
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
@ -217,6 +223,7 @@ struct mapped_device {
|
|||
*/
|
||||
struct dm_md_mempools {
|
||||
mempool_t *io_pool;
|
||||
mempool_t *rq_pool;
|
||||
struct bio_set *bs;
|
||||
};
|
||||
|
||||
|
|
@ -231,6 +238,7 @@ struct table_device {
|
|||
#define RESERVED_MAX_IOS 1024
|
||||
static struct kmem_cache *_io_cache;
|
||||
static struct kmem_cache *_rq_tio_cache;
|
||||
static struct kmem_cache *_rq_cache;
|
||||
|
||||
/*
|
||||
* Bio-based DM's mempools' reserved IOs set by the user.
|
||||
|
|
@ -288,9 +296,14 @@ static int __init local_init(void)
|
|||
if (!_rq_tio_cache)
|
||||
goto out_free_io_cache;
|
||||
|
||||
_rq_cache = kmem_cache_create("dm_clone_request", sizeof(struct request),
|
||||
__alignof__(struct request), 0, NULL);
|
||||
if (!_rq_cache)
|
||||
goto out_free_rq_tio_cache;
|
||||
|
||||
r = dm_uevent_init();
|
||||
if (r)
|
||||
goto out_free_rq_tio_cache;
|
||||
goto out_free_rq_cache;
|
||||
|
||||
deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
|
||||
if (!deferred_remove_workqueue) {
|
||||
|
|
@ -312,6 +325,8 @@ out_free_workqueue:
|
|||
destroy_workqueue(deferred_remove_workqueue);
|
||||
out_uevent_exit:
|
||||
dm_uevent_exit();
|
||||
out_free_rq_cache:
|
||||
kmem_cache_destroy(_rq_cache);
|
||||
out_free_rq_tio_cache:
|
||||
kmem_cache_destroy(_rq_tio_cache);
|
||||
out_free_io_cache:
|
||||
|
|
@ -325,6 +340,7 @@ static void local_exit(void)
|
|||
flush_scheduled_work();
|
||||
destroy_workqueue(deferred_remove_workqueue);
|
||||
|
||||
kmem_cache_destroy(_rq_cache);
|
||||
kmem_cache_destroy(_rq_tio_cache);
|
||||
kmem_cache_destroy(_io_cache);
|
||||
unregister_blkdev(_major, _name);
|
||||
|
|
@ -577,6 +593,17 @@ static void free_rq_tio(struct dm_rq_target_io *tio)
|
|||
mempool_free(tio, tio->md->io_pool);
|
||||
}
|
||||
|
||||
static struct request *alloc_clone_request(struct mapped_device *md,
|
||||
gfp_t gfp_mask)
|
||||
{
|
||||
return mempool_alloc(md->rq_pool, gfp_mask);
|
||||
}
|
||||
|
||||
static void free_clone_request(struct mapped_device *md, struct request *rq)
|
||||
{
|
||||
mempool_free(rq, md->rq_pool);
|
||||
}
|
||||
|
||||
static int md_in_flight(struct mapped_device *md)
|
||||
{
|
||||
return atomic_read(&md->pending[READ]) +
|
||||
|
|
@ -992,7 +1019,7 @@ static void end_clone_bio(struct bio *clone, int error)
|
|||
* the md may be freed in dm_put() at the end of this function.
|
||||
* Or do dm_get() before calling this function and dm_put() later.
|
||||
*/
|
||||
static void rq_completed(struct mapped_device *md, int rw, int run_queue)
|
||||
static void rq_completed(struct mapped_device *md, int rw, bool run_queue)
|
||||
{
|
||||
atomic_dec(&md->pending[rw]);
|
||||
|
||||
|
|
@ -1020,12 +1047,17 @@ static void free_rq_clone(struct request *clone)
|
|||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
|
||||
blk_rq_unprep_clone(clone);
|
||||
if (clone->q && clone->q->mq_ops)
|
||||
tio->ti->type->release_clone_rq(clone);
|
||||
else
|
||||
free_clone_request(tio->md, clone);
|
||||
free_rq_tio(tio);
|
||||
}
|
||||
|
||||
/*
|
||||
* Complete the clone and the original request.
|
||||
* Must be called without queue lock.
|
||||
* Must be called without clone's queue lock held,
|
||||
* see end_clone_request() for more details.
|
||||
*/
|
||||
static void dm_end_request(struct request *clone, int error)
|
||||
{
|
||||
|
|
@ -1054,23 +1086,23 @@ static void dm_end_request(struct request *clone, int error)
|
|||
|
||||
static void dm_unprep_request(struct request *rq)
|
||||
{
|
||||
struct request *clone = rq->special;
|
||||
struct dm_rq_target_io *tio = rq->special;
|
||||
struct request *clone = tio->clone;
|
||||
|
||||
rq->special = NULL;
|
||||
rq->cmd_flags &= ~REQ_DONTPREP;
|
||||
|
||||
free_rq_clone(clone);
|
||||
if (clone)
|
||||
free_rq_clone(clone);
|
||||
}
|
||||
|
||||
/*
|
||||
* Requeue the original request of a clone.
|
||||
*/
|
||||
void dm_requeue_unmapped_request(struct request *clone)
|
||||
static void dm_requeue_unmapped_original_request(struct mapped_device *md,
|
||||
struct request *rq)
|
||||
{
|
||||
int rw = rq_data_dir(clone);
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
struct mapped_device *md = tio->md;
|
||||
struct request *rq = tio->orig;
|
||||
int rw = rq_data_dir(rq);
|
||||
struct request_queue *q = rq->q;
|
||||
unsigned long flags;
|
||||
|
||||
|
|
@ -1080,9 +1112,15 @@ void dm_requeue_unmapped_request(struct request *clone)
|
|||
blk_requeue_request(q, rq);
|
||||
spin_unlock_irqrestore(q->queue_lock, flags);
|
||||
|
||||
rq_completed(md, rw, 0);
|
||||
rq_completed(md, rw, false);
|
||||
}
|
||||
|
||||
static void dm_requeue_unmapped_request(struct request *clone)
|
||||
{
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
|
||||
dm_requeue_unmapped_original_request(tio->md, tio->orig);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
|
||||
|
||||
static void __stop_queue(struct request_queue *q)
|
||||
{
|
||||
|
|
@ -1151,8 +1189,15 @@ static void dm_done(struct request *clone, int error, bool mapped)
|
|||
static void dm_softirq_done(struct request *rq)
|
||||
{
|
||||
bool mapped = true;
|
||||
struct request *clone = rq->completion_data;
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
struct dm_rq_target_io *tio = rq->special;
|
||||
struct request *clone = tio->clone;
|
||||
|
||||
if (!clone) {
|
||||
blk_end_request_all(rq, tio->error);
|
||||
rq_completed(tio->md, rq_data_dir(rq), false);
|
||||
free_rq_tio(tio);
|
||||
return;
|
||||
}
|
||||
|
||||
if (rq->cmd_flags & REQ_FAILED)
|
||||
mapped = false;
|
||||
|
|
@ -1164,13 +1209,11 @@ static void dm_softirq_done(struct request *rq)
|
|||
* Complete the clone and the original request with the error status
|
||||
* through softirq context.
|
||||
*/
|
||||
static void dm_complete_request(struct request *clone, int error)
|
||||
static void dm_complete_request(struct request *rq, int error)
|
||||
{
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
struct request *rq = tio->orig;
|
||||
struct dm_rq_target_io *tio = rq->special;
|
||||
|
||||
tio->error = error;
|
||||
rq->completion_data = clone;
|
||||
blk_complete_request(rq);
|
||||
}
|
||||
|
||||
|
|
@ -1178,40 +1221,40 @@ static void dm_complete_request(struct request *clone, int error)
|
|||
* Complete the not-mapped clone and the original request with the error status
|
||||
* through softirq context.
|
||||
* Target's rq_end_io() function isn't called.
|
||||
* This may be used when the target's map_rq() function fails.
|
||||
* This may be used when the target's map_rq() or clone_and_map_rq() functions fail.
|
||||
*/
|
||||
void dm_kill_unmapped_request(struct request *clone, int error)
|
||||
static void dm_kill_unmapped_request(struct request *rq, int error)
|
||||
{
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
struct request *rq = tio->orig;
|
||||
|
||||
rq->cmd_flags |= REQ_FAILED;
|
||||
dm_complete_request(clone, error);
|
||||
dm_complete_request(rq, error);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
|
||||
|
||||
/*
|
||||
* Called with the queue lock held
|
||||
* Called with the clone's queue lock held
|
||||
*/
|
||||
static void end_clone_request(struct request *clone, int error)
|
||||
{
|
||||
/*
|
||||
* For just cleaning up the information of the queue in which
|
||||
* the clone was dispatched.
|
||||
* The clone is *NOT* freed actually here because it is alloced from
|
||||
* dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
|
||||
*/
|
||||
__blk_put_request(clone->q, clone);
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
|
||||
if (!clone->q->mq_ops) {
|
||||
/*
|
||||
* For just cleaning up the information of the queue in which
|
||||
* the clone was dispatched.
|
||||
* The clone is *NOT* freed actually here because it is alloced
|
||||
* from dm own mempool (REQ_ALLOCED isn't set).
|
||||
*/
|
||||
__blk_put_request(clone->q, clone);
|
||||
}
|
||||
|
||||
/*
|
||||
* Actual request completion is done in a softirq context which doesn't
|
||||
* hold the queue lock. Otherwise, deadlock could occur because:
|
||||
* hold the clone's queue lock. Otherwise, deadlock could occur because:
|
||||
* - another request may be submitted by the upper level driver
|
||||
* of the stacking during the completion
|
||||
* - the submission which requires queue lock may be done
|
||||
* against this queue
|
||||
* against this clone's queue
|
||||
*/
|
||||
dm_complete_request(clone, error);
|
||||
dm_complete_request(tio->orig, error);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1689,19 +1732,19 @@ static void dm_request(struct request_queue *q, struct bio *bio)
|
|||
_dm_request(q, bio);
|
||||
}
|
||||
|
||||
void dm_dispatch_request(struct request *rq)
|
||||
static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
|
||||
{
|
||||
int r;
|
||||
|
||||
if (blk_queue_io_stat(rq->q))
|
||||
rq->cmd_flags |= REQ_IO_STAT;
|
||||
if (blk_queue_io_stat(clone->q))
|
||||
clone->cmd_flags |= REQ_IO_STAT;
|
||||
|
||||
rq->start_time = jiffies;
|
||||
r = blk_insert_cloned_request(rq->q, rq);
|
||||
clone->start_time = jiffies;
|
||||
r = blk_insert_cloned_request(clone->q, clone);
|
||||
if (r)
|
||||
/* must complete clone in terms of original request */
|
||||
dm_complete_request(rq, r);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(dm_dispatch_request);
|
||||
|
||||
static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
|
||||
void *data)
|
||||
|
|
@ -1718,12 +1761,11 @@ static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
|
|||
}
|
||||
|
||||
static int setup_clone(struct request *clone, struct request *rq,
|
||||
struct dm_rq_target_io *tio)
|
||||
struct dm_rq_target_io *tio, gfp_t gfp_mask)
|
||||
{
|
||||
int r;
|
||||
|
||||
blk_rq_init(NULL, clone);
|
||||
r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
|
||||
r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,
|
||||
dm_rq_bio_constructor, tio);
|
||||
if (r)
|
||||
return r;
|
||||
|
|
@ -1734,14 +1776,37 @@ static int setup_clone(struct request *clone, struct request *rq,
|
|||
clone->end_io = end_clone_request;
|
||||
clone->end_io_data = tio;
|
||||
|
||||
tio->clone = clone;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct request *clone_rq(struct request *rq, struct mapped_device *md,
|
||||
gfp_t gfp_mask)
|
||||
struct dm_rq_target_io *tio, gfp_t gfp_mask)
|
||||
{
|
||||
struct request *clone = alloc_clone_request(md, gfp_mask);
|
||||
|
||||
if (!clone)
|
||||
return NULL;
|
||||
|
||||
blk_rq_init(NULL, clone);
|
||||
if (setup_clone(clone, rq, tio, gfp_mask)) {
|
||||
/* -ENOMEM */
|
||||
free_clone_request(md, clone);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
static void map_tio_request(struct kthread_work *work);
|
||||
|
||||
static struct dm_rq_target_io *prep_tio(struct request *rq,
|
||||
struct mapped_device *md, gfp_t gfp_mask)
|
||||
{
|
||||
struct request *clone;
|
||||
struct dm_rq_target_io *tio;
|
||||
int srcu_idx;
|
||||
struct dm_table *table;
|
||||
|
||||
tio = alloc_rq_tio(md, gfp_mask);
|
||||
if (!tio)
|
||||
|
|
@ -1749,18 +1814,23 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
|
|||
|
||||
tio->md = md;
|
||||
tio->ti = NULL;
|
||||
tio->clone = NULL;
|
||||
tio->orig = rq;
|
||||
tio->error = 0;
|
||||
memset(&tio->info, 0, sizeof(tio->info));
|
||||
init_kthread_work(&tio->work, map_tio_request);
|
||||
|
||||
clone = &tio->clone;
|
||||
if (setup_clone(clone, rq, tio)) {
|
||||
/* -ENOMEM */
|
||||
free_rq_tio(tio);
|
||||
return NULL;
|
||||
table = dm_get_live_table(md, &srcu_idx);
|
||||
if (!dm_table_mq_request_based(table)) {
|
||||
if (!clone_rq(rq, md, tio, gfp_mask)) {
|
||||
dm_put_live_table(md, srcu_idx);
|
||||
free_rq_tio(tio);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
dm_put_live_table(md, srcu_idx);
|
||||
|
||||
return clone;
|
||||
return tio;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1769,18 +1839,18 @@ static struct request *clone_rq(struct request *rq, struct mapped_device *md,
|
|||
static int dm_prep_fn(struct request_queue *q, struct request *rq)
|
||||
{
|
||||
struct mapped_device *md = q->queuedata;
|
||||
struct request *clone;
|
||||
struct dm_rq_target_io *tio;
|
||||
|
||||
if (unlikely(rq->special)) {
|
||||
DMWARN("Already has something in rq->special.");
|
||||
return BLKPREP_KILL;
|
||||
}
|
||||
|
||||
clone = clone_rq(rq, md, GFP_ATOMIC);
|
||||
if (!clone)
|
||||
tio = prep_tio(rq, md, GFP_ATOMIC);
|
||||
if (!tio)
|
||||
return BLKPREP_DEFER;
|
||||
|
||||
rq->special = clone;
|
||||
rq->special = tio;
|
||||
rq->cmd_flags |= REQ_DONTPREP;
|
||||
|
||||
return BLKPREP_OK;
|
||||
|
|
@ -1788,17 +1858,36 @@ static int dm_prep_fn(struct request_queue *q, struct request *rq)
|
|||
|
||||
/*
|
||||
* Returns:
|
||||
* 0 : the request has been processed (not requeued)
|
||||
* !0 : the request has been requeued
|
||||
* 0 : the request has been processed
|
||||
* DM_MAPIO_REQUEUE : the original request needs to be requeued
|
||||
* < 0 : the request was completed due to failure
|
||||
*/
|
||||
static int map_request(struct dm_target *ti, struct request *clone,
|
||||
static int map_request(struct dm_target *ti, struct request *rq,
|
||||
struct mapped_device *md)
|
||||
{
|
||||
int r, requeued = 0;
|
||||
struct dm_rq_target_io *tio = clone->end_io_data;
|
||||
int r;
|
||||
struct dm_rq_target_io *tio = rq->special;
|
||||
struct request *clone = NULL;
|
||||
|
||||
if (tio->clone) {
|
||||
clone = tio->clone;
|
||||
r = ti->type->map_rq(ti, clone, &tio->info);
|
||||
} else {
|
||||
r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
|
||||
if (r < 0) {
|
||||
/* The target wants to complete the I/O */
|
||||
dm_kill_unmapped_request(rq, r);
|
||||
return r;
|
||||
}
|
||||
if (IS_ERR(clone))
|
||||
return DM_MAPIO_REQUEUE;
|
||||
if (setup_clone(clone, rq, tio, GFP_KERNEL)) {
|
||||
/* -ENOMEM */
|
||||
ti->type->release_clone_rq(clone);
|
||||
return DM_MAPIO_REQUEUE;
|
||||
}
|
||||
}
|
||||
|
||||
tio->ti = ti;
|
||||
r = ti->type->map_rq(ti, clone, &tio->info);
|
||||
switch (r) {
|
||||
case DM_MAPIO_SUBMITTED:
|
||||
/* The target has taken the I/O to submit by itself later */
|
||||
|
|
@ -1806,13 +1895,12 @@ static int map_request(struct dm_target *ti, struct request *clone,
|
|||
case DM_MAPIO_REMAPPED:
|
||||
/* The target has remapped the I/O so dispatch it */
|
||||
trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
|
||||
blk_rq_pos(tio->orig));
|
||||
dm_dispatch_request(clone);
|
||||
blk_rq_pos(rq));
|
||||
dm_dispatch_clone_request(clone, rq);
|
||||
break;
|
||||
case DM_MAPIO_REQUEUE:
|
||||
/* The target wants to requeue the I/O */
|
||||
dm_requeue_unmapped_request(clone);
|
||||
requeued = 1;
|
||||
break;
|
||||
default:
|
||||
if (r > 0) {
|
||||
|
|
@ -1821,20 +1909,27 @@ static int map_request(struct dm_target *ti, struct request *clone,
|
|||
}
|
||||
|
||||
/* The target wants to complete the I/O */
|
||||
dm_kill_unmapped_request(clone, r);
|
||||
break;
|
||||
dm_kill_unmapped_request(rq, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
return requeued;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
|
||||
static void map_tio_request(struct kthread_work *work)
|
||||
{
|
||||
struct request *clone;
|
||||
struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work);
|
||||
struct request *rq = tio->orig;
|
||||
struct mapped_device *md = tio->md;
|
||||
|
||||
if (map_request(tio->ti, rq, md) == DM_MAPIO_REQUEUE)
|
||||
dm_requeue_unmapped_original_request(md, rq);
|
||||
}
|
||||
|
||||
static void dm_start_request(struct mapped_device *md, struct request *orig)
|
||||
{
|
||||
blk_start_request(orig);
|
||||
clone = orig->special;
|
||||
atomic_inc(&md->pending[rq_data_dir(clone)]);
|
||||
atomic_inc(&md->pending[rq_data_dir(orig)]);
|
||||
|
||||
/*
|
||||
* Hold the md reference here for the in-flight I/O.
|
||||
|
|
@ -1844,8 +1939,6 @@ static struct request *dm_start_request(struct mapped_device *md, struct request
|
|||
* See the comment in rq_completed() too.
|
||||
*/
|
||||
dm_get(md);
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1858,7 +1951,8 @@ static void dm_request_fn(struct request_queue *q)
|
|||
int srcu_idx;
|
||||
struct dm_table *map = dm_get_live_table(md, &srcu_idx);
|
||||
struct dm_target *ti;
|
||||
struct request *rq, *clone;
|
||||
struct request *rq;
|
||||
struct dm_rq_target_io *tio;
|
||||
sector_t pos;
|
||||
|
||||
/*
|
||||
|
|
@ -1880,34 +1974,29 @@ static void dm_request_fn(struct request_queue *q)
|
|||
ti = dm_table_find_target(map, pos);
|
||||
if (!dm_target_is_valid(ti)) {
|
||||
/*
|
||||
* Must perform setup, that dm_done() requires,
|
||||
* Must perform setup, that rq_completed() requires,
|
||||
* before calling dm_kill_unmapped_request
|
||||
*/
|
||||
DMERR_LIMIT("request attempted access beyond the end of device");
|
||||
clone = dm_start_request(md, rq);
|
||||
dm_kill_unmapped_request(clone, -EIO);
|
||||
dm_start_request(md, rq);
|
||||
dm_kill_unmapped_request(rq, -EIO);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ti->type->busy && ti->type->busy(ti))
|
||||
goto delay_and_out;
|
||||
|
||||
clone = dm_start_request(md, rq);
|
||||
|
||||
spin_unlock(q->queue_lock);
|
||||
if (map_request(ti, clone, md))
|
||||
goto requeued;
|
||||
dm_start_request(md, rq);
|
||||
|
||||
tio = rq->special;
|
||||
/* Establish tio->ti before queuing work (map_tio_request) */
|
||||
tio->ti = ti;
|
||||
queue_kthread_work(&md->kworker, &tio->work);
|
||||
BUG_ON(!irqs_disabled());
|
||||
spin_lock(q->queue_lock);
|
||||
}
|
||||
|
||||
goto out;
|
||||
|
||||
requeued:
|
||||
BUG_ON(!irqs_disabled());
|
||||
spin_lock(q->queue_lock);
|
||||
|
||||
delay_and_out:
|
||||
blk_delay_queue(q, HZ / 10);
|
||||
out:
|
||||
|
|
@ -2093,6 +2182,7 @@ static struct mapped_device *alloc_dev(int minor)
|
|||
INIT_WORK(&md->work, dm_wq_work);
|
||||
init_waitqueue_head(&md->eventq);
|
||||
init_completion(&md->kobj_holder.completion);
|
||||
md->kworker_task = NULL;
|
||||
|
||||
md->disk->major = _major;
|
||||
md->disk->first_minor = minor;
|
||||
|
|
@ -2153,8 +2243,13 @@ static void free_dev(struct mapped_device *md)
|
|||
unlock_fs(md);
|
||||
bdput(md->bdev);
|
||||
destroy_workqueue(md->wq);
|
||||
|
||||
if (md->kworker_task)
|
||||
kthread_stop(md->kworker_task);
|
||||
if (md->io_pool)
|
||||
mempool_destroy(md->io_pool);
|
||||
if (md->rq_pool)
|
||||
mempool_destroy(md->rq_pool);
|
||||
if (md->bs)
|
||||
bioset_free(md->bs);
|
||||
blk_integrity_unregister(md->disk);
|
||||
|
|
@ -2188,23 +2283,24 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
|
|||
bioset_free(md->bs);
|
||||
md->bs = p->bs;
|
||||
p->bs = NULL;
|
||||
} else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
|
||||
/*
|
||||
* There's no need to reload with request-based dm
|
||||
* because the size of front_pad doesn't change.
|
||||
* Note for future: If you are to reload bioset,
|
||||
* prep-ed requests in the queue may refer
|
||||
* to bio from the old bioset, so you must walk
|
||||
* through the queue to unprep.
|
||||
*/
|
||||
}
|
||||
/*
|
||||
* There's no need to reload with request-based dm
|
||||
* because the size of front_pad doesn't change.
|
||||
* Note for future: If you are to reload bioset,
|
||||
* prep-ed requests in the queue may refer
|
||||
* to bio from the old bioset, so you must walk
|
||||
* through the queue to unprep.
|
||||
*/
|
||||
goto out;
|
||||
}
|
||||
|
||||
BUG_ON(!p || md->io_pool || md->bs);
|
||||
BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
|
||||
|
||||
md->io_pool = p->io_pool;
|
||||
p->io_pool = NULL;
|
||||
md->rq_pool = p->rq_pool;
|
||||
p->rq_pool = NULL;
|
||||
md->bs = p->bs;
|
||||
p->bs = NULL;
|
||||
|
||||
|
|
@ -2407,6 +2503,14 @@ unsigned dm_get_md_type(struct mapped_device *md)
|
|||
return md->type;
|
||||
}
|
||||
|
||||
static bool dm_md_type_request_based(struct mapped_device *md)
|
||||
{
|
||||
unsigned table_type = dm_get_md_type(md);
|
||||
|
||||
return (table_type == DM_TYPE_REQUEST_BASED ||
|
||||
table_type == DM_TYPE_MQ_REQUEST_BASED);
|
||||
}
|
||||
|
||||
struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
|
||||
{
|
||||
return md->immutable_target_type;
|
||||
|
|
@ -2444,6 +2548,11 @@ static int dm_init_request_based_queue(struct mapped_device *md)
|
|||
blk_queue_prep_rq(md->queue, dm_prep_fn);
|
||||
blk_queue_lld_busy(md->queue, dm_lld_busy);
|
||||
|
||||
/* Also initialize the request-based DM worker thread */
|
||||
init_kthread_worker(&md->kworker);
|
||||
md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,
|
||||
"kdmwork-%s", dm_device_name(md));
|
||||
|
||||
elv_register_queue(md->queue);
|
||||
|
||||
return 1;
|
||||
|
|
@ -2454,8 +2563,7 @@ static int dm_init_request_based_queue(struct mapped_device *md)
|
|||
*/
|
||||
int dm_setup_md_queue(struct mapped_device *md)
|
||||
{
|
||||
if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
|
||||
!dm_init_request_based_queue(md)) {
|
||||
if (dm_md_type_request_based(md) && !dm_init_request_based_queue(md)) {
|
||||
DMWARN("Cannot initialize queue for request-based mapped device");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
|
@ -2534,6 +2642,9 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
|
|||
set_bit(DMF_FREEING, &md->flags);
|
||||
spin_unlock(&_minor_lock);
|
||||
|
||||
if (dm_request_based(md))
|
||||
flush_kthread_worker(&md->kworker);
|
||||
|
||||
if (!dm_suspended_md(md)) {
|
||||
dm_table_presuspend_targets(map);
|
||||
dm_table_postsuspend_targets(map);
|
||||
|
|
@ -2777,8 +2888,10 @@ static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
|
|||
* Stop md->queue before flushing md->wq in case request-based
|
||||
* dm defers requests to md->wq from md->queue.
|
||||
*/
|
||||
if (dm_request_based(md))
|
||||
if (dm_request_based(md)) {
|
||||
stop_queue(md->queue);
|
||||
flush_kthread_worker(&md->kworker);
|
||||
}
|
||||
|
||||
flush_workqueue(md->wq);
|
||||
|
||||
|
|
@ -3124,24 +3237,35 @@ struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, u
|
|||
{
|
||||
struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
|
||||
struct kmem_cache *cachep;
|
||||
unsigned int pool_size;
|
||||
unsigned int pool_size = 0;
|
||||
unsigned int front_pad;
|
||||
|
||||
if (!pools)
|
||||
return NULL;
|
||||
|
||||
if (type == DM_TYPE_BIO_BASED) {
|
||||
switch (type) {
|
||||
case DM_TYPE_BIO_BASED:
|
||||
cachep = _io_cache;
|
||||
pool_size = dm_get_reserved_bio_based_ios();
|
||||
front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
|
||||
} else if (type == DM_TYPE_REQUEST_BASED) {
|
||||
cachep = _rq_tio_cache;
|
||||
break;
|
||||
case DM_TYPE_REQUEST_BASED:
|
||||
pool_size = dm_get_reserved_rq_based_ios();
|
||||
pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
|
||||
if (!pools->rq_pool)
|
||||
goto out;
|
||||
/* fall through to setup remaining rq-based pools */
|
||||
case DM_TYPE_MQ_REQUEST_BASED:
|
||||
cachep = _rq_tio_cache;
|
||||
if (!pool_size)
|
||||
pool_size = dm_get_reserved_rq_based_ios();
|
||||
front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
|
||||
/* per_bio_data_size is not used. See __bind_mempools(). */
|
||||
WARN_ON(per_bio_data_size != 0);
|
||||
} else
|
||||
break;
|
||||
default:
|
||||
goto out;
|
||||
}
|
||||
|
||||
pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
|
||||
if (!pools->io_pool)
|
||||
|
|
@ -3170,6 +3294,9 @@ void dm_free_md_mempools(struct dm_md_mempools *pools)
|
|||
if (pools->io_pool)
|
||||
mempool_destroy(pools->io_pool);
|
||||
|
||||
if (pools->rq_pool)
|
||||
mempool_destroy(pools->rq_pool);
|
||||
|
||||
if (pools->bs)
|
||||
bioset_free(pools->bs);
|
||||
|
||||
|
|
|
|||
|
|
@ -34,9 +34,10 @@
|
|||
/*
|
||||
* Type of table and mapped_device's mempool
|
||||
*/
|
||||
#define DM_TYPE_NONE 0
|
||||
#define DM_TYPE_BIO_BASED 1
|
||||
#define DM_TYPE_REQUEST_BASED 2
|
||||
#define DM_TYPE_NONE 0
|
||||
#define DM_TYPE_BIO_BASED 1
|
||||
#define DM_TYPE_REQUEST_BASED 2
|
||||
#define DM_TYPE_MQ_REQUEST_BASED 3
|
||||
|
||||
/*
|
||||
* List of devices that a metadevice uses and should open/close.
|
||||
|
|
@ -73,6 +74,7 @@ int dm_table_any_busy_target(struct dm_table *t);
|
|||
unsigned dm_table_get_type(struct dm_table *t);
|
||||
struct target_type *dm_table_get_immutable_target_type(struct dm_table *t);
|
||||
bool dm_table_request_based(struct dm_table *t);
|
||||
bool dm_table_mq_request_based(struct dm_table *t);
|
||||
void dm_table_free_md_mempools(struct dm_table *t);
|
||||
struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t);
|
||||
|
||||
|
|
@ -99,7 +101,8 @@ int dm_setup_md_queue(struct mapped_device *md);
|
|||
/*
|
||||
* To check whether the target type is request-based or not (bio-based).
|
||||
*/
|
||||
#define dm_target_request_based(t) ((t)->type->map_rq != NULL)
|
||||
#define dm_target_request_based(t) (((t)->type->map_rq != NULL) || \
|
||||
((t)->type->clone_and_map_rq != NULL))
|
||||
|
||||
/*
|
||||
* To check whether the target type is a hybrid (capable of being
|
||||
|
|
|
|||
|
|
@ -48,6 +48,11 @@ typedef void (*dm_dtr_fn) (struct dm_target *ti);
|
|||
typedef int (*dm_map_fn) (struct dm_target *ti, struct bio *bio);
|
||||
typedef int (*dm_map_request_fn) (struct dm_target *ti, struct request *clone,
|
||||
union map_info *map_context);
|
||||
typedef int (*dm_clone_and_map_request_fn) (struct dm_target *ti,
|
||||
struct request *rq,
|
||||
union map_info *map_context,
|
||||
struct request **clone);
|
||||
typedef void (*dm_release_clone_request_fn) (struct request *clone);
|
||||
|
||||
/*
|
||||
* Returns:
|
||||
|
|
@ -143,6 +148,8 @@ struct target_type {
|
|||
dm_dtr_fn dtr;
|
||||
dm_map_fn map;
|
||||
dm_map_request_fn map_rq;
|
||||
dm_clone_and_map_request_fn clone_and_map_rq;
|
||||
dm_release_clone_request_fn release_clone_rq;
|
||||
dm_endio_fn end_io;
|
||||
dm_request_endio_fn rq_end_io;
|
||||
dm_presuspend_fn presuspend;
|
||||
|
|
@ -600,9 +607,6 @@ static inline unsigned long to_bytes(sector_t n)
|
|||
/*-----------------------------------------------------------------
|
||||
* Helper for block layer and dm core operations
|
||||
*---------------------------------------------------------------*/
|
||||
void dm_dispatch_request(struct request *rq);
|
||||
void dm_requeue_unmapped_request(struct request *rq);
|
||||
void dm_kill_unmapped_request(struct request *rq, int error);
|
||||
int dm_underlying_device_busy(struct request_queue *q);
|
||||
|
||||
#endif /* _LINUX_DEVICE_MAPPER_H */
|
||||
|
|
|
|||
|
|
@ -267,9 +267,9 @@ enum {
|
|||
#define DM_DEV_SET_GEOMETRY _IOWR(DM_IOCTL, DM_DEV_SET_GEOMETRY_CMD, struct dm_ioctl)
|
||||
|
||||
#define DM_VERSION_MAJOR 4
|
||||
#define DM_VERSION_MINOR 29
|
||||
#define DM_VERSION_MINOR 30
|
||||
#define DM_VERSION_PATCHLEVEL 0
|
||||
#define DM_VERSION_EXTRA "-ioctl (2014-10-28)"
|
||||
#define DM_VERSION_EXTRA "-ioctl (2014-12-22)"
|
||||
|
||||
/* Status bits */
|
||||
#define DM_READONLY_FLAG (1 << 0) /* In/Out */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue