9 files changed, 1112 insertions, 438 deletions
diff --git a/curtin/block/bcache.py b/curtin/block/bcache.py
index 852cef23..c31852ee 100644
--- a/curtin/block/bcache.py
+++ b/curtin/block/bcache.py
@@ -1,11 +1,15 @@
 # This file is part of curtin. See LICENSE file for copyright and license info.
 
+import errno
 import os
 
 from curtin import util
 from curtin.log import LOG
 from . import sys_block_path
 
+# Wait up to 20 minutes (150 + 300 + 750 = 1200 seconds)
+BCACHE_RETRIES = [sleep for nap in [1, 2, 5] for sleep in [nap] * 150]
+
 
 def superblock_asdict(device=None, data=None):
     """ Convert output from bcache-super-show into a dictionary"""
@@ -14,7 +18,12 @@ def superblock_asdict(device=None, data=None):
         raise ValueError('Supply a device name, or data to parse')
 
     if not data:
-        data, _err = util.subp(['bcache-super-show', device], capture=True)
+        try:
+            data, _err = util.subp(['bcache-super-show', device], capture=True)
+        except util.ProcessExecutionError as e:
+            LOG.debug('Failed to parse bcache superblock on %s:%s',
+                      device, e)
+            return None
     bcache_super = {}
     for line in data.splitlines():
         if not line:
@@ -25,8 +34,22 @@ def superblock_asdict(device=None, data=None):
     return bcache_super
 
 
-def parse_sb_version(sb_version):
-    """ Convert sb_version string to integer if possible"""
+def parse_sb_version(device=None, sbdict=None):
+    """ Parse bcache 'sb_version' field to integer if possible.
+
+    """
+    if not device and not sbdict:
+        raise ValueError('Supply a device name or bcache superblock dict')
+
+    if not sbdict:
+        sbdict = superblock_asdict(device=device)
+        if not sbdict:
+            LOG.info('Cannot parse sb.version without bcache superblock')
+            return None
+    if not isinstance(sbdict, dict):
+        raise ValueError('Invalid sbdict type, must be dict')
+
+    sb_version = sbdict.get('sb.version')
     try:
         # 'sb.version': '1 [backing device]'
         # 'sb.version': '3 [caching device]'
@@ -34,11 +57,25 @@ def parse_sb_version(sb_version):
     except (AttributeError, ValueError):
         LOG.warning("Failed to parse bcache 'sb.version' field"
                     " as integer: %s", sb_version)
-        return None
+        raise
 
     return version
 
 
+def _check_bcache_type(device, sysfs_attr, sb_version, superblock=False):
+    """ helper for checking bcache type via sysfs or bcache superblock. """
+    if not superblock:
+        if not device.endswith('bcache'):
+            sys_block = os.path.join(sys_block_path(device), 'bcache')
+        else:
+            sys_block = device
+        bcache_sys_attr = os.path.join(sys_block, sysfs_attr)
+        LOG.debug('path exists %s', bcache_sys_attr)
+        return os.path.exists(bcache_sys_attr)
+    else:
+        return parse_sb_version(device=device) == sb_version
+
+
 def is_backing(device, superblock=False):
     """ Test if device is a bcache backing device
 
@@ -47,15 +84,7 @@ def is_backing(device, superblock=False):
 
     However if a device is not active then read the superblock
     of the device and check that sb.version == 1"""
-
-    if not superblock:
-        sys_block = sys_block_path(device)
-        bcache_sys_attr = os.path.join(sys_block, 'bcache', 'label')
-        return os.path.exists(bcache_sys_attr)
-    else:
-        bcache_super = superblock_asdict(device=device)
-        sb_version = parse_sb_version(bcache_super['sb.version'])
-        return bcache_super and sb_version == 1
+    return _check_bcache_type(device, 'label', 1, superblock=superblock)
 
 
 def is_caching(device, superblock=False):
@@ -67,21 +96,171 @@ def is_caching(device, superblock=False):
     However if a device is not active then read the superblock
     of the device and check that sb.version == 3"""
 
-    if not superblock:
-        sys_block = sys_block_path(device)
-        bcache_sysattr = os.path.join(sys_block, 'bcache',
-                                      'cache_replacement_policy')
-        return os.path.exists(bcache_sysattr)
-    else:
-        bcache_super = superblock_asdict(device=device)
-        sb_version = parse_sb_version(bcache_super['sb.version'])
-        return bcache_super and sb_version == 3
+    LOG.debug('Checking if %s is bcache caching device', device)
+    return _check_bcache_type(device, 'cache_replacement_policy', 3,
+                              superblock=superblock)
+
+
+def sysfs_path(device, strict=True):
+    """ Return /sys/class/block/<device>/bcache path for device. """
+    path = os.path.join(sys_block_path(device, strict=strict), 'bcache')
+    if strict and not os.path.exists(path):
+        err = OSError(
+            "device '{}' did not have existing syspath '{}'".format(
+                device, path))
+        err.errno = errno.ENOENT
+        raise err
+
+    return path
 
 
 def write_label(label, device):
     """ write label to bcache device """
-    sys_block = sys_block_path(device)
-    bcache_sys_attr = os.path.join(sys_block, 'bcache', 'label')
-    util.write_file(bcache_sys_attr, content=label)
+    bcache_sys_attr = os.path.join(sysfs_path(device), 'label')
+    util.write_file(bcache_sys_attr, content=label, mode=None)
+
+
+def get_attached_cacheset(device):
+    """  return the sysfs path to an attached cacheset. """
+    bcache_cache = os.path.join(sysfs_path(device), 'cache')
+    if os.path.exists(bcache_cache):
+        return os.path.basename(os.path.realpath(bcache_cache))
+
+    return None
+
+
+def get_cacheset_members(cset_uuid):
+    """ return a list of sysfs paths to backing devices
+        attached to the specified cache set.
+
+    Example:
+      % get_cacheset_members('08307315-48e7-4e46-8742-2ec37d615829')
+      ['/sys/devices/pci0000:00/0000:00:08.0/virtio5/block/vdc/bcache',
+       '/sys/devices/pci0000:00/0000:00:07.0/virtio4/block/vdb/bcache',
+       '/sys/devices/pci0000:00/0000:00:06.0/virtio3/block/vda/vda1/bcache']
+    """
+    cset_path = '/sys/fs/bcache/%s' % cset_uuid
+    members = []
+    if os.path.exists(cset_path):
+        # extract bdev* links
+        bdevs = [link for link in os.listdir(cset_path)
+                 if link.startswith('bdev')]
+        # resolve symlink to target
+        members = [os.path.realpath("%s/%s" % (cset_path, bdev))
+                   for bdev in bdevs]
+
+    return members
+
+
+def get_cacheset_cachedev(cset_uuid):
+    """ Return a sysfs path to a cacheset cache device's bcache dir."""
+
+    # XXX: bcache cachesets only have a single cache0 entry
+    cachedev = '/sys/fs/bcache/%s/cache0' % cset_uuid
+    if os.path.exists(cachedev):
+        return os.path.realpath(cachedev)
+
+    return None
+
+
+def get_backing_device(bcache_kname):
+    """ For a given bcacheN kname, return the backing device
+        bcache sysfs dir.
+
+        bcache0 -> /sys/.../devices/.../device/bcache
+    """
+    bcache_deps = '/sys/class/block/%s/slaves' % bcache_kname
+
+    try:
+        # if the bcache device is deleted, this may fail
+        deps = os.listdir(bcache_deps)
+    except util.FileMissingError as e:
+        LOG.debug('Transient race, bcache slave path not found: %s', e)
+        return None
+
+    # a running bcache device has two entries in slaves, the cacheset
+    # device, and the backing device. There may only be the backing
+    # device (if a bcache device is found but not currently attached
+    # to a cacheset.
+    if len(deps) == 0:
+        raise RuntimeError(
+            '%s unexpected empty dir: %s' % (bcache_kname, bcache_deps))
+
+    for dev in (sysfs_path(dep) for dep in deps):
+        if is_backing(dev):
+            return dev
+
+    return None
+
+
+def stop_cacheset(cset_uuid):
+    """stop specified bcache cacheset."""
+    # we may be called with a full path or just the uuid
+    if cset_uuid.startswith('/sys/fs/bcache/'):
+        cset_device = cset_uuid
+    else:
+        cset_device = "/sys/fs/bcache/%s" % cset_uuid
+    LOG.info('Stopping bcache set device: %s', cset_device)
+    _stop_device(cset_device)
+
+
+def stop_device(device):
+    """Stop the specified bcache device."""
+    if not device.startswith('/sys'):
+        raise ValueError('Invalid device %s, must be sysfs path' % device)
+
+    if not any(f(device) for f in (is_backing, is_caching)):
+        raise ValueError('Cannot stop non-bcache device: %s' % device)
+
+    LOG.debug('Stopping bcache layer on %s', device)
+    _stop_device(device)
+
+
+def _stop_device(device):
+    """  write to sysfs 'stop' and wait for path to be removed
+
+    The caller needs to ensure that supplied path to the device
+    is a 'bcache' sysfs path on a device.  This may be one of the
+    following scenarios:
+
+    Cacheset:
+      /sys/fs/bcache/<uuid>/
+
+    Bcache device:
+     /sys/class/block/bcache0/bcache
+
+    Backing device
+     /sys/class/block/vdb/bcache
+
+    Cached device
+     /sys/class/block/nvme0n1p1/bcache/set
+
+    To support all of these, we append 'stop' to the path
+    and write '1' and then wait for the 'stop' path to
+    be removed.
+    """
+    bcache_stop = os.path.join(device, 'stop')
+    if not os.path.exists(bcache_stop):
+        LOG.debug('bcache._stop_device: already removed %s', bcache_stop)
+        return
+
+    LOG.debug('bcache._stop_device: device=%s stop_path=%s',
+              device, bcache_stop)
+    try:
+        util.write_file(bcache_stop, '1', mode=None)
+    except (IOError, OSError) as e:
+        # Note: if we get any exceptions in the above exception classes
+        # it is a result of attempting to write "1" into the sysfs path
+        # The range of errors changes depending on when we race with
+        # the kernel asynchronously removing the sysfs path. Therefore
+        # we log the exception errno we got, but do not re-raise as
+        # the calling process is watching whether the same sysfs path
+        # is being removed;  if it fails to go away then we'll have
+        # a log of the exceptions to debug.
+        LOG.debug('Error writing to bcache stop file %s, device removed: %s',
+                  bcache_stop, e)
+    finally:
+        util.wait_for_removal(bcache_stop, retries=BCACHE_RETRIES)
+
 
 # vi: ts=4 expandtab syntax=python
diff --git a/curtin/block/clear_holders.py b/curtin/block/clear_holders.py
index 67caee39..fb7fba4a 100644
--- a/curtin/block/clear_holders.py
+++ b/curtin/block/clear_holders.py
@@ -6,13 +6,13 @@ top of a block device, making it possible to reuse the block device without
 having to reboot the system
 """
 
-import errno
 import glob
 import os
 import time
 
 from curtin import (block, udev, util)
 from curtin.swap import is_swap_device
+from curtin.block import bcache
 from curtin.block import lvm
 from curtin.block import mdadm
 from curtin.block import zfs
@@ -47,80 +47,29 @@ def get_dmsetup_uuid(device):
     return out.strip()
 
 
-def get_bcache_using_dev(device, strict=True):
-    """
-    Get the /sys/fs/bcache/ path of the bcache cache device bound to
-    specified device
-    """
-    # FIXME: when block.bcache is written this should be moved there
-    sysfs_path = block.sys_block_path(device)
-    path = os.path.realpath(os.path.join(sysfs_path, 'bcache', 'cache'))
-    if strict and not os.path.exists(path):
-        err = OSError(
-            "device '{}' did not have existing syspath '{}'".format(
-                device, path))
-        err.errno = errno.ENOENT
-        raise err
-
-    return path
-
-
-def get_bcache_sys_path(device, strict=True):
-    """
-    Get the /sys/class/block/<device>/bcache path
-    """
-    sysfs_path = block.sys_block_path(device, strict=strict)
-    path = os.path.join(sysfs_path, 'bcache')
-    if strict and not os.path.exists(path):
-        err = OSError(
-            "device '{}' did not have existing syspath '{}'".format(
-                device, path))
-        err.errno = errno.ENOENT
-        raise err
-
-    return path
-
-
-def maybe_stop_bcache_device(device):
-    """Attempt to stop the provided device_path or raise unexpected errors."""
-    bcache_stop = os.path.join(device, 'stop')
-    try:
-        util.write_file(bcache_stop, '1', mode=None)
-    except (IOError, OSError) as e:
-        # Note: if we get any exceptions in the above exception classes
-        # it is a result of attempting to write "1" into the sysfs path
-        # The range of errors changes depending on when we race with
-        # the kernel asynchronously removing the sysfs path. Therefore
-        # we log the exception errno we got, but do not re-raise as
-        # the calling process is watching whether the same sysfs path
-        # is being removed;  if it fails to go away then we'll have
-        # a log of the exceptions to debug.
-        LOG.debug('Error writing to bcache stop file %s, device removed: %s',
-                  bcache_stop, e)
-
-
 def shutdown_bcache(device):
     """
     Shut down bcache for specified bcache device
 
-    1. Stop the cacheset that `device` is connected to
-    2. Stop the 'device'
+    1. wipe the bcache device contents
+    2. extract the cacheset uuid (if cached)
+    3. extract the backing device
+    4. stop cacheset (if present)
+    5. stop the bcacheN device
+    6. wait for removal of sysfs path to bcacheN, bcacheN/bcache and
+       backing/bcache to go away
     """
     if not device.startswith('/sys/class/block'):
         raise ValueError('Invalid Device (%s): '
                          'Device path must start with /sys/class/block/',
                          device)
 
-    LOG.info('Wiping superblock on bcache device: %s', device)
-    _wipe_superblock(block.sysfs_to_devpath(device), exclusive=False)
-
     # bcache device removal should be fast but in an extreme
     # case, might require the cache device to flush large
     # amounts of data to a backing device.  The strategy here
     # is to wait for approximately 30 seconds but to check
     # frequently since curtin cannot proceed until devices
     # cleared.
-    removal_retries = [0.2] * 150  # 30 seconds total
     bcache_shutdown_message = ('shutdown_bcache running on {} has determined '
                                'that the device has already been shut down '
                                'during handling of another bcache dev. '
@@ -130,60 +79,39 @@ def shutdown_bcache(device):
         LOG.info(bcache_shutdown_message)
         return
 
-    # get slaves [vdb1, vdc], allow for slaves to not have bcache dir
-    try:
-        slave_paths = [get_bcache_sys_path(k, strict=False) for k in
-                       os.listdir(os.path.join(device, 'slaves'))]
-    except util.FileMissingError as e:
-        LOG.debug('Transient race, bcache slave path not found: %s', e)
-        slave_paths = []
-
-    # stop cacheset if it exists
-    bcache_cache_sysfs = get_bcache_using_dev(device, strict=False)
-    if not os.path.exists(bcache_cache_sysfs):
-        LOG.info('bcache cacheset already removed: %s',
-                 os.path.basename(bcache_cache_sysfs))
-    else:
-        LOG.info('stopping bcache cacheset at: %s', bcache_cache_sysfs)
-        maybe_stop_bcache_device(bcache_cache_sysfs)
-        try:
-            util.wait_for_removal(bcache_cache_sysfs, retries=removal_retries)
-        except OSError:
-            LOG.info('Failed to stop bcache cacheset %s', bcache_cache_sysfs)
-            raise
+    LOG.info('Wiping superblock on bcache device: %s', device)
+    _wipe_superblock(block.sysfs_to_devpath(device), exclusive=False)
+
+    # collect required information before stopping bcache device
+    # UUID from /sys/fs/cache/UUID
+    cset_uuid = bcache.get_attached_cacheset(device)
+    # /sys/class/block/vdX which is a backing dev of device (bcacheN)
+    backing_sysfs = bcache.get_backing_device(block.path_to_kname(device))
+    # /sys/class/block/bcacheN/bache
+    bcache_sysfs = bcache.sysfs_path(device, strict=False)
+
+    # stop cacheset if one is presennt
+    if cset_uuid:
+        LOG.info('%s was attached to cacheset %s, stopping cacheset',
+                 device, cset_uuid)
+        bcache.stop_cacheset(cset_uuid)
 
         # let kernel settle before the next remove
         udev.udevadm_settle()
+        LOG.info('bcache cacheset stopped: %s', cset_uuid)
 
-    # after stopping cache set, we may need to stop the device
-    # both the dev and sysfs entry should be gone.
-
-    # we know the bcacheN device is really gone when we've removed:
-    #  /sys/class/block/{bcacheN}
-    #  /sys/class/block/slaveN1/bcache
-    #  /sys/class/block/slaveN2/bcache
-    bcache_block_sysfs = get_bcache_sys_path(device, strict=False)
-    to_check = [device] + slave_paths
+    # test and log whether the device paths are still present
+    to_check = [bcache_sysfs, backing_sysfs]
     found_devs = [os.path.exists(p) for p in to_check]
     LOG.debug('os.path.exists on blockdevs:\n%s',
               list(zip(to_check, found_devs)))
     if not any(found_devs):
         LOG.info('bcache backing device already removed: %s (%s)',
-                 bcache_block_sysfs, device)
-        LOG.debug('bcache slave paths checked: %s', slave_paths)
-        return
+                 bcache_sysfs, device)
+        LOG.debug('bcache backing device checked: %s', backing_sysfs)
     else:
-        LOG.info('stopping bcache backing device at: %s', bcache_block_sysfs)
-        maybe_stop_bcache_device(bcache_block_sysfs)
-        try:
-            # wait for them all to go away
-            for dev in [device, bcache_block_sysfs] + slave_paths:
-                util.wait_for_removal(dev, retries=removal_retries)
-        except OSError:
-            LOG.info('Failed to stop bcache backing device %s',
-                     bcache_block_sysfs)
-            raise
-
+        LOG.info('stopping bcache backing device at: %s', bcache_sysfs)
+        bcache.stop_device(bcache_sysfs)
     return
 
 
@@ -335,10 +263,13 @@ def wipe_superblock(device):
     for bcache_path in ['bcache', 'bcache/set']:
         stop_path = os.path.join(device, bcache_path)
         if os.path.exists(stop_path):
-            LOG.debug('Attempting to release bcache layer from device: %s',
-                      device)
-            maybe_stop_bcache_device(stop_path)
-            continue
+            LOG.debug('Attempting to release bcache layer from device: %s:%s',
+                      device, stop_path)
+            if stop_path.endswith('set'):
+                rp = os.path.realpath(stop_path)
+                bcache.stop_cacheset(rp)
+            else:
+                bcache._stop_device(stop_path)
 
     _wipe_superblock(blockdev)
 
@@ -537,8 +468,10 @@ def plan_shutdown_holder_trees(holders_trees):
     for holders_tree in holders_trees:
         flatten_holders_tree(holders_tree)
 
-    # return list of entry dicts with highest level first
-    return [reg[k] for k in sorted(reg, key=lambda x: reg[x]['level'] * -1)]
+    # return list of entry dicts with highest level first, then dev_type
+    return [reg[k]
+            for k in sorted(reg, key=lambda x: (reg[x]['level'] * -1,
+                                                reg[x]['dev_type']))]
 
 
 def format_holders_tree(holders_tree):
diff --git a/examples/tests/bcache-ceph-nvme.yaml b/examples/tests/bcache-ceph-nvme.yaml
new file mode 100644
index 00000000..507bc0ec
--- /dev/null
+++ b/examples/tests/bcache-ceph-nvme.yaml
@@ -0,0 +1,227 @@
+install:
+  unmount: disabled
+showtrace: true
+storage:
+  config:
+  - grub_device: true
+    id: sda
+    model: MG04SCA60EA
+    name: sda
+    ptable: gpt
+    serial: '500003986840e04d'
+    type: disk
+    wipe: superblock
+  - id: sdb
+    model: MG04SCA60EA
+    name: sdb
+    serial: '500003986833378d'
+    type: disk
+    wipe: superblock
+  - id: sdc
+    model: MG04SCA60EA
+    name: sdc
+    serial: '5000039868108f0d'
+    type: disk
+    wipe: superblock
+  - id: sdd
+    model: MG04SCA60EA
+    name: sdd
+    serial: '5000039868107619'
+    type: disk
+    wipe: superblock
+  - id: sde
+    model: MG04SCA60EA
+    name: sde
+    serial: '5000039868418549'
+    type: disk
+    wipe: superblock
+  - id: sdf
+    model: SAMSUNG MZ7LM240
+    name: sdf
+    ptable: gpt
+    serial: 'S3LKNX0K202278'
+    type: disk
+    wipe: superblock
+  - id: sdg
+    model: MG04SCA60EA
+    name: sdg
+    serial: '5000039868333799'
+    type: disk
+    wipe: superblock
+  - id: sdh
+    model: SAMSUNG MZ7LM240
+    name: sdh
+    ptable: gpt
+    serial: 'S3LKNX0K200071'
+    type: disk
+    wipe: superblock
+  - id: nvme0n1
+    model: UCSC-NVME-H32003
+    name: nvme0n1
+    ptable: gpt
+    serial: nvme-SDM000014FB6
+    type: disk
+    wipe: superblock
+  - id: nvme1n1
+    model: UCSC-NVME-H32003
+    name: nvme1n1
+    ptable: gpt
+    serial: nvme-SDM000014F3C
+    type: disk
+    wipe: superblock
+  - device: sda
+    id: sda-part1
+    name: sda-part1
+    number: 1
+    offset: 4194304B
+    size: 5G
+    type: partition
+    uuid: 11d66990-9b49-4fe5-b933-d8f1527023d3
+    wipe: superblock
+  - device: sdf
+    id: sdf-part1
+    name: sdf-part1
+    number: 1
+    offset: 4194304B
+    size: 5G
+    type: partition
+    uuid: e86f3316-aacc-4958-a6db-34875a5fde7c
+    wipe: superblock
+  - device: sdf
+    id: sdf-part2
+    name: sdf-part2
+    number: 2
+    size: 5G
+    type: partition
+    uuid: aa5d9117-de31-4311-9bf1-28ae45e9748f
+    wipe: superblock
+  - device: sdf
+    id: sdf-part3
+    name: sdf-part3
+    number: 3
+    size: 5G
+    type: partition
+    uuid: a312bb83-e34a-4d05-b45e-006d2f4291ee
+    wipe: superblock
+  - device: sdh
+    id: sdh-part1
+    name: sdh-part1
+    number: 1
+    offset: 4194304B
+    size: 5G
+    type: partition
+    uuid: a15f79c9-4277-4c58-8a68-65a6f59864f3
+    wipe: superblock
+  - devices:
+    - sdf-part3
+    - sdh-part1
+    id: md0
+    name: md0
+    raidlevel: 1
+    spare_devices: []
+    type: raid
+  - device: nvme0n1
+    id: nvme0n1-part1
+    name: nvme0n1-part1
+    number: 1
+    offset: 4194304B
+    size: 4G
+    type: partition
+    uuid: 5a406a80-dd85-4f5a-83a5-9dd0bf27cb6e
+    wipe: superblock
+  - device: nvme0n1
+    id: nvme0n1-part2
+    name: nvme0n1-part2
+    number: 2
+    size: 4G
+    type: partition
+    uuid: a1ab6ecb-e4b1-44eb-b895-949808741ab3
+    wipe: superblock
+  - backing_device: sda-part1
+    cache_device: nvme0n1-part2
+    cache_mode: writeback
+    id: osddata0
+    name: osddata0
+    type: bcache
+  - backing_device: sdc
+    cache_device: nvme0n1-part2
+    cache_mode: writeback
+    id: osddata2
+    name: osddata2
+    type: bcache
+  - backing_device: sdb
+    cache_device: nvme0n1-part2
+    cache_mode: writeback
+    id: osddata1
+    name: osddata1
+    type: bcache
+  - device: nvme1n1
+    id: nvme1n1-part1
+    name: nvme1n1-part1
+    number: 1
+    offset: 4194304B
+    size: 4G
+    type: partition
+    uuid: fa904f69-2de7-43c6-a9b6-14b4e7139ce7
+    wipe: superblock
+  - device: nvme1n1
+    id: nvme1n1-part2
+    name: nvme1n1-part2
+    number: 2
+    size: 4G
+    type: partition
+    uuid: 2f5e22d5-6737-4ad2-94ff-e0cf7ef8c97c
+    wipe: superblock
+  - backing_device: sdg
+    cache_device: nvme1n1-part2
+    cache_mode: writeback
+    id: osddata5
+    name: osddata5
+    type: bcache
+  - backing_device: sde
+    cache_device: nvme1n1-part2
+    cache_mode: writeback
+    id: osddata4
+    name: osddata4
+    type: bcache
+  - backing_device: sdd
+    cache_device: nvme1n1-part2
+    cache_mode: writeback
+    id: osddata3
+    name: osddata3
+    type: bcache
+  - fstype: fat32
+    id: sdf-part1_format
+    label: ''
+    type: format
+    uuid: 7ddf7d92-5e9f-4347-93e2-b34455339342
+    volume: sdf-part1
+  - fstype: ext4
+    id: sdf-part2_format
+    label: ''
+    type: format
+    uuid: 771ea4e9-873c-48ab-9ac6-e49ede275019
+    volume: sdf-part2
+  - fstype: ext4
+    id: md0_format
+    label: os
+    type: format
+    uuid: b29b461c-34f0-4d22-9454-0034e34b1b5c
+    volume: md0
+  - device: md0_format
+    id: md0_mount
+    options: ''
+    path: /
+    type: mount
+  - device: sdf-part2_format
+    id: sdf-part2_mount
+    options: ''
+    path: /boot
+    type: mount
+  - device: sdf-part1_format
+    id: sdf-part1_mount
+    options: ''
+    path: /boot/efi
+    type: mount
+  version: 1
+verbosity: 3
diff --git a/examples/tests/dirty_disks_config.yaml b/examples/tests/dirty_disks_config.yaml
index fb9a0d68..bcf3fbc9 100644
--- a/examples/tests/dirty_disks_config.yaml
+++ b/examples/tests/dirty_disks_config.yaml
@@ -52,6 +52,22 @@ bucket:
       done
       # remove any existing metadata written from early disk config
       rm -f /etc/mdadm/mdadm.conf
+  - &naptime  |
+      #!/bin/sh
+      # This function attempts to settle and flush IO to devices
+      # in some scenarios vmtest devices have had lots of IO
+      # sent to them and they've yet to consume it all. Give it a
+      # chance to flush themselves
+      echo "VMTEST: io flush nap time, 12 second cleanse"
+      sleep 3
+      sync; sync; sync;
+      sleep 3
+      echo 3 > /proc/sys/vm/drop_caches
+      sleep 3
+      sync; sync; sync;
+      sleep 3
+      echo "VMTEST: io flush nap time complete;"
+
 
 early_commands:
   # running block-meta custom from the install environment
@@ -61,9 +77,10 @@ early_commands:
   # that could unintentionally mess things up.
   01-blockmeta: [env, -u, OUTPUT_FSTAB,
               TARGET_MOUNT_POINT=/tmp/my.bdir/target,
-              WORKING_DIR=/tmp/my.bdir/work.d, 
+              WORKING_DIR=/tmp/my.bdir/work.d,
               curtin, --showtrace, -v, block-meta, --umount, custom]
   02-enable_swaps: [sh, -c, *swapon]
   03-disable_rpool: [sh, -c, *zpool_export]
   04-lvm_stop: [sh, -c, *lvm_stop]
   05-mdadm_stop: [sh, -c, *mdadm_stop]
+  06-naptime: [sh, -c, *naptime]
diff --git a/tests/data/bcache-super-show-backing b/tests/data/bcache-super-show-backing
new file mode 100644
index 00000000..03debdf1
--- /dev/null
+++ b/tests/data/bcache-super-show-backing
@@ -0,0 +1,14 @@
+sb.magic		ok
+sb.first_sector		8 [match]
+sb.csum			B92908820E241EDD [match]
+sb.version		1 [backing device]
+
+dev.label		(empty)
+dev.uuid		f36394c0-3cc0-4423-8d6f-ffac130f171a
+dev.sectors_per_block	1
+dev.sectors_per_bucket	1024
+dev.data.first_sector	16
+dev.data.cache_mode	1 [writeback]
+dev.data.cache_state	2 [dirty]
+
+cset.uuid		01da3829-ea92-4600-bd40-7f95974f3087
diff --git a/tests/data/bcache-super-show-caching b/tests/data/bcache-super-show-caching
new file mode 100644
index 00000000..9125700f
--- /dev/null
+++ b/tests/data/bcache-super-show-caching
@@ -0,0 +1,18 @@
+sb.magic		ok
+sb.first_sector		8 [match]
+sb.csum			2F8BB7E8DC53E0B6 [match]
+sb.version		3 [cache device]
+
+dev.label		(empty)
+dev.uuid		ff51a56d-eddc-41b3-867d-8744277c5281
+dev.sectors_per_block	1
+dev.sectors_per_bucket	1024
+dev.cache.first_sector	1024
+dev.cache.cache_sectors	234372096
+dev.cache.total_sectors	234373120
+dev.cache.ordered	yes
+dev.cache.discard	no
+dev.cache.pos		0
+dev.cache.replacement	0 [lru]
+
+cset.uuid		01da3829-ea92-4600-bd40-7f95974f3087
diff --git a/tests/unittests/test_block_bcache.py b/tests/unittests/test_block_bcache.py
new file mode 100644
index 00000000..79365223
--- /dev/null
+++ b/tests/unittests/test_block_bcache.py
@@ -0,0 +1,448 @@
+import mock
+import os
+
+from curtin.block import bcache
+from curtin.util import (FileMissingError, load_file, ProcessExecutionError)
+from .helpers import CiTestCase
+
+
+class TestBlockBcache(CiTestCase):
+
+    def setUp(self):
+        super(TestBlockBcache, self).setUp()
+        self.add_patch('curtin.block.bcache.util.subp', 'mock_subp')
+
+    def _datafile(self, name):
+        path = 'tests/data'
+        return os.path.join(path, name)
+
+    expected = {
+        'backing': {
+            "cset.uuid": "01da3829-ea92-4600-bd40-7f95974f3087",
+            "dev.data.cache_mode": "1 [writeback]",
+            "dev.data.cache_state": "2 [dirty]",
+            "dev.data.first_sector": "16",
+            "dev.label": "(empty)",
+            "dev.sectors_per_block": "1",
+            "dev.sectors_per_bucket": "1024",
+            "dev.uuid": "f36394c0-3cc0-4423-8d6f-ffac130f171a",
+            "sb.csum": "B92908820E241EDD [match]",
+            "sb.first_sector": "8 [match]",
+            "sb.magic": "ok",
+            "sb.version": "1 [backing device]"
+        },
+        'caching': {
+            "cset.uuid": "01da3829-ea92-4600-bd40-7f95974f3087",
+            "dev.cache.cache_sectors": "234372096",
+            "dev.cache.discard": "no",
+            "dev.cache.first_sector": "1024",
+            "dev.cache.ordered": "yes",
+            "dev.cache.pos": "0",
+            "dev.cache.replacement": "0 [lru]",
+            "dev.cache.total_sectors": "234373120",
+            "dev.label": "(empty)",
+            "dev.sectors_per_block": "1",
+            "dev.sectors_per_bucket": "1024",
+            "dev.uuid": "ff51a56d-eddc-41b3-867d-8744277c5281",
+            "sb.csum": "2F8BB7E8DC53E0B6 [match]",
+            "sb.first_sector": "8 [match]",
+            "sb.magic": "ok",
+            "sb.version": "3 [cache device]"
+        },
+    }
+
+    @mock.patch('curtin.block.bcache.util.subp')
+    def test_superblock_asdict(self, m_subp):
+        """ verify parsing bcache-super-show matches expected results."""
+        device = self.random_string()
+        results = {}
+        prefix = 'bcache-super-show-'
+        scenarios = ['backing', 'caching']
+
+        # XXX: Parameterize me
+        for superblock in scenarios:
+            datafile = prefix + superblock
+            contents = load_file(self._datafile(datafile))
+            m_subp.return_value = (contents, '')
+
+            results[superblock] = bcache.superblock_asdict(device=device)
+
+        for superblock in scenarios:
+            comment = 'mismatch in %s' % superblock
+            self.assertDictEqual(
+                self.expected[superblock], results[superblock], comment)
+
+    def test_superblock_asdict_no_dev_no_data(self):
+        """ superblock_asdict raises ValueError without device or data."""
+        with self.assertRaises(ValueError):
+            bcache.superblock_asdict()
+
+    @mock.patch('curtin.block.bcache.util.subp')
+    def test_superblock_asdict_calls_bcache_super_show(self, m_subp):
+        """ superblock_asdict calls bcache-super-show on device."""
+        device = self.random_string()
+        m_subp.return_value = ('', '')
+        bcache.superblock_asdict(device=device)
+        m_subp.assert_called_with(['bcache-super-show', device], capture=True)
+
+    @mock.patch('curtin.block.bcache.util.subp')
+    def test_superblock_asdict_does_not_call_subp_with_data(self, m_subp):
+        """ superblock_asdict does not bcache-super-show with data provided."""
+        key = self.random_string()
+        value = self.random_string()
+        mydata = "\t".join([key, value])
+        result = bcache.superblock_asdict(data=mydata)
+        self.assertEqual({key: value}, result)
+        m_subp.assert_not_called()
+
+    @mock.patch('curtin.block.bcache.util.subp')
+    def test_superblock_asdict_returns_none_invalid_superblock(self, m_subp):
+        device = self.random_string()
+        m_subp.side_effect = ProcessExecutionError(stdout=self.random_string(),
+                                                   stderr=self.random_string(),
+                                                   exit_code=2)
+        self.assertEqual(None, bcache.superblock_asdict(device=device))
+
+    def test_parse_sb_version(self):
+        """ parse_sb_version converts sb.version field into integer value. """
+        sbdict = {'sb.version': '1 [backing device]'}
+        self.assertEqual(1, bcache.parse_sb_version(sbdict=sbdict))
+
+    # XXX: Parameterize me
+    def test_parse_sb_version_raises_exceptions_on_garbage_dict(self):
+        """ parse_sb_version raises Exceptions on garbage dicts."""
+        with self.assertRaises(AttributeError):
+            bcache.parse_sb_version(sbdict={self.random_string():
+                                            self.random_string()})
+
+    # XXX: Parameterize me
+    def test_parse_sb_version_raises_exceptions_on_non_dict(self):
+        """ parse_sb_version raises Exceptions on non-dict input."""
+        with self.assertRaises(ValueError):
+            bcache.parse_sb_version(sbdict=self.random_string())
+
+    @mock.patch('curtin.block.bcache.superblock_asdict')
+    def test_is_backing_superblock(self, m_sbdict):
+        """ is_backing returns True when given backing superblock dict. """
+        bdict = {'sb.version': '1 [backing device]'}
+        m_sbdict.return_value = bdict
+        self.assertEqual(True, bcache.is_backing(self.random_string(),
+                                                 superblock=True))
+
+    @mock.patch('curtin.block.bcache.superblock_asdict')
+    def test_is_backing_superblock_invalid(self, m_sbdict):
+        """ is_backing returns False when parsing invalid superblock. """
+        m_sbdict.return_value = None
+        self.assertEqual(False, bcache.is_backing(self.random_string(),
+                                                  superblock=True))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_is_backing_sysfs(self, m_sysb_path, m_path_exists):
+        """ is_backing returns True if sysfs path has bcache/label. """
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = True
+        self.assertEqual(True, bcache.is_backing(kname))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_is_backing_sysfs_false(self, m_sysb_path, m_path_exists):
+        """ is_backing returns False if path does not have bcache/label. """
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = False
+        self.assertEqual(False, bcache.is_backing(kname))
+
+    @mock.patch('curtin.block.bcache.superblock_asdict')
+    def test_is_cacheing_superblock(self, m_sbdict):
+        """ is_caching returns True when given caching superblock dict. """
+        bdict = {'sb.version': '3 [caching device]'}
+        m_sbdict.return_value = bdict
+        self.assertEqual(True, bcache.is_caching(self.random_string(),
+                                                 superblock=True))
+
+    @mock.patch('curtin.block.bcache.superblock_asdict')
+    def test_is_caching_superblock_invalid(self, m_sbdict):
+        """ is_caching returns False when parsing invalid superblock. """
+        m_sbdict.return_value = None
+        self.assertEqual(False, bcache.is_caching(self.random_string(),
+                                                  superblock=True))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_is_caching_sysfs(self, m_sysb_path, m_path_exists):
+        """ is_caching returns True if sysfs path has bcache/label. """
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = True
+        self.assertEqual(True, bcache.is_caching(kname))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_is_caching_sysfs_false(self, m_sysb_path, m_path_exists):
+        """ is_caching returns False if path does not have bcache/label. """
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = False
+        self.assertEqual(False, bcache.is_caching(kname))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_sysfs_path(self, m_sysb_path, m_path_exists):
+        """ sysfs_path returns /sys/class/block/<device>/bcache for device."""
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = True
+        self.assertEqual('/sys/class/block/%s/bcache' % kname,
+                         bcache.sysfs_path(kname))
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_sysfs_path_raise_strict_nopath(self, m_sysb_path, m_path_exists):
+        """ sysfs_path raises OSError on strict=True and missing path. """
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = False
+        with self.assertRaises(OSError):
+            bcache.sysfs_path(kname)
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sys_block_path')
+    def test_sysfs_path_non_strict(self, m_sysb_path, m_path_exists):
+        """ sysfs_path returns path if missing and strict=False."""
+        kname = self.random_string()
+        m_sysb_path.return_value = '/sys/class/block/%s' % kname
+        m_path_exists.return_value = False
+        self.assertEqual('/sys/class/block/%s/bcache' % kname,
+                         bcache.sysfs_path(kname, strict=False))
+
+    @mock.patch('curtin.block.bcache.sysfs_path')
+    @mock.patch('curtin.block.bcache.util.write_file')
+    def test_write_label(self, m_write_file, m_sysfs_path):
+        """ write_label writes label to device/bcache/label attribute."""
+        label = self.random_string()
+        kname = self.random_string()
+        bdir = '/sys/class/block/%s/bcache' % kname
+        label_path = bdir + '/label'
+        m_sysfs_path.return_value = bdir
+        bcache.write_label(label, kname)
+        m_write_file.assert_called_with(label_path, content=label, mode=None)
+
+    @mock.patch('curtin.block.bcache.os.path.realpath')
+    @mock.patch('curtin.block.bcache.os.path.basename')
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.sysfs_path')
+    def test_get_attached_cacheset(self, m_spath, m_exists, m_base, m_real):
+        """ get_attached_cacheset resolves 'cache' symlink under bcache dir."""
+        kname = self.random_string()
+        cset_uuid = self.random_string()
+        bdir = '/sys/class/block/%s/bcache' % kname
+        m_spath.return_value = bdir
+        m_exists.return_value = True
+        cacheset = '/sys/fs/bcache/%s' % cset_uuid
+        m_base.return_value = cacheset
+
+        self.assertEqual(cacheset, bcache.get_attached_cacheset(kname))
+        m_exists.assert_called_with(bdir + '/cache')
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.os.path.realpath')
+    @mock.patch('curtin.block.bcache.os.listdir')
+    def test_get_cacheset_members(self, m_listdir, m_real, m_exists):
+        """ get_cacheset_members finds backing devices using cacheset."""
+        cset_uuid = self.random_string()
+        bdev_target = self.random_string()
+        cset_dir_keys = [
+            'average_key_size', 'bdev0', 'block_size', 'btree_cache_size',
+            'bucket_size', 'cache0', 'cache_available_percent', 'clear_stats',
+            'congested', 'congested_read_threshold_us',
+            'congested_write_threshold_us', 'errors', 'flash_vol_create',
+            'internal', 'io_error_halflife', 'io_error_limit',
+            'journal_delay_ms', 'root_usage_percent', 'stats_day',
+            'stats_five_minute', 'stats_hour', 'stats_total', 'stop',
+            'synchronous', 'tree_depth', 'unregister',
+        ]
+        cset_path = '/sys/fs/bcache/%s' % cset_uuid
+        m_listdir.return_value = cset_dir_keys
+        m_real.side_effect = iter([bdev_target])
+        m_exists.return_value = True
+        results = bcache.get_cacheset_members(cset_uuid)
+        self.assertEqual([bdev_target], results)
+        m_listdir.assert_called_with(cset_path)
+
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    @mock.patch('curtin.block.bcache.os.path.realpath')
+    def test_get_cacheset_cachedev(self, m_real, m_exists):
+        """ get_cacheset_cachedev finds cacheset device path."""
+        cset_uuid = self.random_string()
+        cachedev_target = self.random_string()
+        cset_path = '/sys/fs/bcache/%s/cache0' % cset_uuid
+        m_exists.return_value = True
+        m_real.side_effect = iter([cachedev_target])
+        results = bcache.get_cacheset_cachedev(cset_uuid)
+        self.assertEqual(cachedev_target, results)
+        m_real.assert_called_with(cset_path)
+
+    @mock.patch('curtin.block.bcache.is_backing')
+    @mock.patch('curtin.block.bcache.sysfs_path')
+    @mock.patch('curtin.block.bcache.os.listdir')
+    def test_get_backing_device(self, m_list, m_sysp, m_back):
+        """ extract sysfs path to backing device from bcache kname."""
+        bcache_kname = self.random_string()
+        backing_kname = self.random_string()
+        caching_kname = self.random_string()
+        m_list.return_value = [backing_kname, caching_kname]
+        m_sysp.side_effect = lambda x: '/sys/class/block/%s/bcache' % x
+        m_back.side_effect = iter([True, False])
+
+        self.assertEqual('/sys/class/block/%s/bcache' % backing_kname,
+                         bcache.get_backing_device(bcache_kname))
+
+    @mock.patch('curtin.block.bcache.is_backing')
+    @mock.patch('curtin.block.bcache.sysfs_path')
+    @mock.patch('curtin.block.bcache.os.listdir')
+    def test_get_backing_device_none_empty_dir(self, m_list, m_sysp, m_back):
+        """ get_backing_device returns None on missing deps dir. """
+        bcache_kname = self.random_string()
+        m_list.side_effect = FileMissingError('does not exist')
+        self.assertEqual(None, bcache.get_backing_device(bcache_kname))
+
+    @mock.patch('curtin.block.bcache.is_backing')
+    @mock.patch('curtin.block.bcache.sysfs_path')
+    @mock.patch('curtin.block.bcache.os.listdir')
+    def test_get_backing_device_raise_empty_dir(self, m_list, m_sysp, m_back):
+        """ get_backing_device raises RuntimeError on empty deps dir."""
+        bcache_kname = self.random_string()
+        m_list.return_value = []
+        with self.assertRaises(RuntimeError):
+            bcache.get_backing_device(bcache_kname)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    def test_stop_cacheset(self, m_stop):
+        """ stop_cacheset calls _stop_device with correct sysfs path. """
+        cset_uuid = self.random_string()
+        bcache.stop_cacheset(cset_uuid)
+        m_stop.assert_called_with('/sys/fs/bcache/%s' % cset_uuid)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    def test_stop_cacheset_full_path(self, m_stop):
+        """ stop_cacheset accepts full path to cacheset. """
+        cset_path = '/sys/fs/bcache/%s' % self.random_string()
+        bcache.stop_cacheset(cset_path)
+        m_stop.assert_called_with(cset_path)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    @mock.patch('curtin.block.bcache.is_caching')
+    @mock.patch('curtin.block.bcache.is_backing')
+    def test_stop_device_backing(self, m_back, m_cache, m_stop):
+        """ stop_device allows backing device to be stopped. """
+        device = '/sys/class/block/%s' % self.random_string()
+        m_back.return_value = True
+        bcache.stop_device(device)
+        m_stop.assert_called_with(device)
+        m_back.assert_called_with(device)
+        self.assertEqual(0, m_cache.call_count)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    @mock.patch('curtin.block.bcache.is_caching')
+    @mock.patch('curtin.block.bcache.is_backing')
+    def test_stop_device_caching(self, m_back, m_cache, m_stop):
+        """ stop_device allows caching device to be stopped. """
+        device = '/sys/class/block/%s' % self.random_string()
+        m_back.return_value = False
+        m_cache.return_value = True
+        bcache.stop_device(device)
+        m_stop.assert_called_with(device)
+        m_back.assert_called_with(device)
+        m_cache.assert_called_with(device)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    @mock.patch('curtin.block.bcache.is_caching')
+    @mock.patch('curtin.block.bcache.is_backing')
+    def test_stop_device_raise_non_syspath(self, m_back, m_cache, m_stop):
+        """ stop_device raises ValueError if device is not sysfs path."""
+        device = self.random_string()
+        with self.assertRaises(ValueError):
+            bcache.stop_device(device)
+        self.assertEqual(0, m_stop.call_count)
+        self.assertEqual(0, m_back.call_count)
+        self.assertEqual(0, m_cache.call_count)
+
+    @mock.patch('curtin.block.bcache._stop_device')
+    @mock.patch('curtin.block.bcache.is_caching')
+    @mock.patch('curtin.block.bcache.is_backing')
+    def test_stop_device_raise_non_bcache_dev(self, m_back, m_cache, m_stop):
+        """ stop_device raises ValueError if device is not bcache device."""
+        device = '/sys/class/block/%s' % self.random_string()
+        m_back.return_value = False
+        m_cache.return_value = False
+        with self.assertRaises(ValueError):
+            bcache.stop_device(device)
+        self.assertEqual(0, m_stop.call_count)
+        self.assertEqual(1, m_back.call_count)
+        self.assertEqual(1, m_cache.call_count)
+
+    @mock.patch('curtin.block.bcache.util.wait_for_removal')
+    @mock.patch('curtin.block.bcache.util.write_file')
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    def test__stop_device_stops_bcache_devs(self, m_exists, m_write, m_wait):
+        """ _stop_device accepts  path and issue stop."""
+        device = self.random_string()
+        stop_path = os.path.join(device, 'stop')
+        m_exists.return_value = True
+        bcache._stop_device(device)
+        m_exists.assert_called_with(stop_path)
+        m_write.assert_called_with(stop_path, '1', mode=None)
+        m_wait.assert_called_with(stop_path, retries=bcache.BCACHE_RETRIES)
+
+    @mock.patch('curtin.block.bcache.util.wait_for_removal')
+    @mock.patch('curtin.block.bcache.util.write_file')
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    def test__stop_device_already_removed(self, m_exists, m_write, m_wait):
+        """ _stop_device skips if device path is missing. """
+        device = self.random_string()
+        stop_path = os.path.join(device, 'stop')
+        m_exists.return_value = False
+
+        bcache._stop_device(device)
+        m_exists.assert_called_with(stop_path)
+        self.assertEqual(0, m_write.call_count)
+        self.assertEqual(0, m_wait.call_count)
+
+    @mock.patch('curtin.block.bcache.util.wait_for_removal')
+    @mock.patch('curtin.block.bcache.util.write_file')
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    def test__stop_device_eats_err_calls_wait(self, m_exists, m_write, m_wait):
+        """ _stop_device eats IOError or OSErrors wait still called"""
+        device = self.random_string()
+        stop_path = os.path.join(device, 'stop')
+        m_exists.return_value = True
+        m_write.side_effect = IOError('permission denied')
+
+        bcache._stop_device(device)
+
+        m_exists.assert_called_with(stop_path)
+        m_write.assert_called_with(stop_path, '1', mode=None)
+        m_wait.assert_called_with(stop_path, retries=bcache.BCACHE_RETRIES)
+
+    @mock.patch('curtin.block.bcache.util.wait_for_removal')
+    @mock.patch('curtin.block.bcache.util.write_file')
+    @mock.patch('curtin.block.bcache.os.path.exists')
+    def test__stop_device_raises_if_wait_expires(self, m_exists, m_write,
+                                                 m_wait):
+        """ _stop_device raises OSError if wait time expires """
+        device = self.random_string()
+        stop_path = os.path.join(device, 'stop')
+        m_exists.return_value = True
+        m_wait.side_effect = (
+            OSError('Timeout exeeded for removal of %s' % stop_path))
+        with self.assertRaises(OSError):
+            bcache._stop_device(device)
+
+        m_exists.assert_called_with(stop_path)
+        m_write.assert_called_with(stop_path, '1', mode=None)
+        m_wait.assert_called_with(stop_path, retries=bcache.BCACHE_RETRIES)
+
+
+# vi: ts=4 expandtab syntax=python
diff --git a/tests/unittests/test_clear_holders.py b/tests/unittests/test_clear_holders.py
index 10ea1abc..e4d8f8d6 100644
--- a/tests/unittests/test_clear_holders.py
+++ b/tests/unittests/test_clear_holders.py
@@ -1,9 +1,9 @@
 # This file is part of curtin. See LICENSE file for copyright and license info.
 
-import errno
 import mock
 import os
 import textwrap
+import uuid
 
 from curtin.block import clear_holders
 from curtin.util import ProcessExecutionError
@@ -89,30 +89,6 @@ class TestClearHolders(CiTestCase):
         self.assertEqual(res, uuid)
         mock_block.sysfs_to_devpath.assert_called_with(self.test_syspath)
 
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.os')
-    def test_get_bcache_using_dev(self, mock_os, mock_block):
-        """Ensure that get_bcache_using_dev works"""
-        fake_bcache = '/sys/fs/bcache/fake'
-        mock_os.path.join.side_effect = os.path.join
-        mock_block.sys_block_path.return_value = self.test_syspath
-        mock_os.path.realpath.return_value = fake_bcache
-
-        bcache_dir = clear_holders.get_bcache_using_dev(self.test_blockdev)
-        mock_os.path.realpath.assert_called_with(self.test_syspath +
-                                                 '/bcache/cache')
-        self.assertEqual(bcache_dir, fake_bcache)
-
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.block')
-    def test_get_bcache_sys_path(self, mock_block, mock_os):
-        fake_backing = '/sys/class/block/fake'
-        mock_block.sys_block_path.return_value = fake_backing
-        mock_os.path.join.side_effect = os.path.join
-        mock_os.path.exists.return_value = True
-        bcache_dir = clear_holders.get_bcache_sys_path("/dev/fake")
-        self.assertEqual(bcache_dir, fake_backing + "/bcache")
-
     @mock.patch('curtin.block.clear_holders.get_dmsetup_uuid')
     @mock.patch('curtin.block.clear_holders.block')
     def test_differentiate_lvm_and_crypt(
@@ -133,264 +109,68 @@ class TestClearHolders(CiTestCase):
             mock_block.path_to_kname.assert_called_with(self.test_syspath)
             mock_get_dmsetup_uuid.assert_called_with(self.test_syspath)
 
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.udev.udevadm_settle')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
     @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache(self, mock_get_bcache, mock_log, mock_os,
-                             mock_util, mock_get_bcache_block,
-                             mock_udevadm_settle, mock_block):
+    @mock.patch('curtin.block.clear_holders.util')
+    @mock.patch('curtin.block.clear_holders.udev')
+    @mock.patch('curtin.block.clear_holders.block')
+    @mock.patch('curtin.block.clear_holders.bcache')
+    def test_shutdown_bcache(self, m_bcache, m_block, m_udev, m_util, m_os):
         """test clear_holders.shutdown_bcache"""
-        #
-        # pass in a sysfs path to a bcache block device,
-        # determine the bcache cset it is part of (or not)
-        # 1) stop the cset device (if it's enabled)
-        # 2) wait on cset to be removed if it was present
-        # 3) stop the block device (if it's still present after stopping cset)
-        # 4) wait on bcache block device to be removed
-        #
-
         device = self.test_syspath
-        mock_block.sys_block_path.return_value = self.test_blockdev
-        bcache_cset_uuid = 'c08ae789-a964-46fb-a66e-650f0ae78f94'
-
-        mock_os.path.exists.return_value = True
-        mock_os.path.join.side_effect = os.path.join
-        # os.path.realpath on symlink of /sys/class/block/null/bcache/cache ->
-        # to /sys/fs/bcache/cset_UUID
-        mock_get_bcache.return_value = '/sys/fs/bcache/' + bcache_cset_uuid
-        mock_get_bcache_block.return_value = device + '/bcache'
+        backing_dev = 'backing1'
+        backing_sys = '/sys/class/block/%s' % backing_dev
+        m_block.sysfs_to_devpath.return_value = self.test_blockdev
+        cset_uuid = str(uuid.uuid4())
+
+        def my_sysblock(p, strict=False):
+            return '/sys/class/block/%s' % os.path.basename(p)
+
+        def my_bsys(p, strict=False):
+            return my_sysblock(p, strict=strict) + '/bcache'
+
+        m_bcache.sysfs_path.side_effect = my_bsys
+        m_os.path.join.side_effect = os.path.join
+        m_os.listdir.return_value = [backing_dev]
+        m_os.path.exists.return_value = True
+        m_bcache.get_attached_cacheset.return_value = cset_uuid
+        m_block.path_to_kname.return_value = self.test_blockdev
+        m_bcache.get_backing_device.return_value = backing_sys + '/bcache'
+        m_bcache.get_cacheset_members.return_value = []
 
         clear_holders.shutdown_bcache(device)
 
-        mock_get_bcache.assert_called_with(device, strict=False)
-        mock_get_bcache_block.assert_called_with(device, strict=False)
-
-        self.assertTrue(mock_log.info.called)
-        self.assertFalse(mock_log.warn.called)
-        mock_util.wait_for_removal.assert_has_calls([
-                mock.call('/sys/fs/bcache/' + bcache_cset_uuid,
-                          retries=self.remove_retries),
-                mock.call(device, retries=self.remove_retries)])
-
-        mock_util.write_file.assert_has_calls([
-                mock.call('/sys/fs/bcache/%s/stop' % bcache_cset_uuid,
-                          '1', mode=None),
-                mock.call(device + '/bcache/stop',
-                          '1', mode=None)])
-
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_non_sysfs_device(self, mock_get_bcache, mock_log,
-                                              mock_os, mock_util,
-                                              mock_get_bcache_block):
-        with self.assertRaises(ValueError):
-            clear_holders.shutdown_bcache(self.test_blockdev)
-
-        self.assertEqual(0, len(mock_get_bcache.call_args_list))
-        self.assertEqual(0, len(mock_log.call_args_list))
-        self.assertEqual(0, len(mock_os.call_args_list))
-        self.assertEqual(0, len(mock_util.call_args_list))
-        self.assertEqual(0, len(mock_get_bcache_block.call_args_list))
-
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_no_device(self, mock_get_bcache, mock_log,
-                                       mock_os, mock_util,
-                                       mock_get_bcache_block, mock_block):
-        mock_block.sysfs_to_devpath.return_value = self.test_blockdev
-        mock_os.path.exists.return_value = False
-
-        clear_holders.shutdown_bcache(self.test_syspath)
-
-        self.assertEqual(3, len(mock_log.info.call_args_list))
-        self.assertEqual(1, len(mock_os.path.exists.call_args_list))
-        self.assertEqual(0, len(mock_get_bcache.call_args_list))
-        self.assertEqual(0, len(mock_util.call_args_list))
-        self.assertEqual(0, len(mock_get_bcache_block.call_args_list))
-
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_no_cset(self, mock_get_bcache, mock_log,
-                                     mock_os, mock_util,
-                                     mock_get_bcache_block, mock_block):
-        mock_block.sysfs_to_devpath.return_value = self.test_blockdev
-        mock_os.path.exists.side_effect = iter([
-                True,   # backing device exists
-                False,  # cset device not present (already removed)
-                True,   # backing device (still) exists
+        # 1. wipe the bcache device contents
+        m_block.wipe_volume.assert_called_with(self.test_blockdev,
+                                               mode='superblock',
+                                               exclusive=False,
+                                               strict=True)
+        # 2. extract the backing device
+        m_bcache.get_backing_device.assert_called_with(self.test_blockdev)
+        m_bcache.sysfs_path.assert_has_calls([
+            mock.call(self.test_syspath, strict=False),
         ])
-        mock_get_bcache.return_value = '/sys/fs/bcache/fake'
-        mock_get_bcache_block.return_value = self.test_syspath + '/bcache'
-        mock_os.path.join.side_effect = os.path.join
-
-        clear_holders.shutdown_bcache(self.test_syspath)
 
-        self.assertEqual(4, len(mock_log.info.call_args_list))
-        self.assertEqual(3, len(mock_os.path.exists.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache_block.call_args_list))
-        self.assertEqual(1, len(mock_util.write_file.call_args_list))
-        self.assertEqual(2, len(mock_util.wait_for_removal.call_args_list))
-
-        mock_get_bcache.assert_called_with(self.test_syspath, strict=False)
-        mock_get_bcache_block.assert_called_with(self.test_syspath,
-                                                 strict=False)
-        mock_util.write_file.assert_called_with(
-            self.test_syspath + '/bcache/stop', '1', mode=None)
-        retries = self.remove_retries
-        mock_util.wait_for_removal.assert_has_calls([
-            mock.call(self.test_syspath, retries=retries),
-            mock.call(self.test_syspath + '/bcache', retries=retries)])
+        # 3. extract the cacheset uuid
+        m_bcache.get_attached_cacheset.assert_called_with(device)
 
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.udev.udevadm_settle')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_delete_cset_and_backing(self, mock_get_bcache,
-                                                     mock_log, mock_os,
-                                                     mock_util,
-                                                     mock_get_bcache_block,
-                                                     mock_udevadm_settle,
-                                                     mock_block):
-        mock_block.sysfs_to_devpath.return_value = self.test_blockdev
-        mock_os.path.exists.side_effect = iter([
-                True,  # backing device exists
-                True,  # cset device not present (already removed)
-                True,  # backing device (still) exists
-        ])
-        cset = '/sys/fs/bcache/fake'
-        mock_get_bcache.return_value = cset
-        mock_get_bcache_block.return_value = self.test_syspath + '/bcache'
-        mock_os.path.join.side_effect = os.path.join
+        # 4. stop the cacheset
+        m_bcache.stop_cacheset.assert_called_with(cset_uuid)
 
-        clear_holders.shutdown_bcache(self.test_syspath)
+        # 5. stop the bcacheN device
+        m_bcache.stop_device.assert_any_call(my_bsys(device))
 
-        self.assertEqual(4, len(mock_log.info.call_args_list))
-        self.assertEqual(3, len(mock_os.path.exists.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache_block.call_args_list))
-        self.assertEqual(2, len(mock_util.write_file.call_args_list))
-        self.assertEqual(3, len(mock_util.wait_for_removal.call_args_list))
-
-        mock_get_bcache.assert_called_with(self.test_syspath, strict=False)
-        mock_get_bcache_block.assert_called_with(self.test_syspath,
-                                                 strict=False)
-        mock_util.write_file.assert_has_calls([
-            mock.call(cset + '/stop', '1', mode=None),
-            mock.call(self.test_syspath + '/bcache/stop', '1', mode=None)])
-        mock_util.wait_for_removal.assert_has_calls([
-            mock.call(cset, retries=self.remove_retries),
-            mock.call(self.test_syspath, retries=self.remove_retries)
-        ])
+    def test_shutdown_bcache_non_sysfs_device(self):
+        """ raises ValueError if called on non-bcache device."""
+        with self.assertRaises(ValueError):
+            clear_holders.shutdown_bcache(self.test_blockdev)
 
-    @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.clear_holders.udev.udevadm_settle')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
     @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_delete_cset_no_backing(self, mock_get_bcache,
-                                                    mock_log, mock_os,
-                                                    mock_util,
-                                                    mock_get_bcache_block,
-                                                    mock_udevadm_settle,
-                                                    mock_block):
-        mock_block.sysfs_to_devpath.return_value = self.test_blockdev
-        mock_os.path.exists.side_effect = iter([
-                True,   # backing device exists
-                True,   # cset device not present (already removed)
-                False,  # backing device is removed with cset
-        ])
-        cset = '/sys/fs/bcache/fake'
-        mock_get_bcache.return_value = cset
-        mock_get_bcache_block.return_value = self.test_syspath + '/bcache'
-        mock_os.path.join.side_effect = os.path.join
-
-        clear_holders.shutdown_bcache(self.test_syspath)
-
-        self.assertEqual(4, len(mock_log.info.call_args_list))
-        self.assertEqual(3, len(mock_os.path.exists.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache_block.call_args_list))
-        self.assertEqual(1, len(mock_util.write_file.call_args_list))
-        self.assertEqual(1, len(mock_util.wait_for_removal.call_args_list))
-
-        mock_get_bcache.assert_called_with(self.test_syspath, strict=False)
-        mock_util.write_file.assert_has_calls([
-            mock.call(cset + '/stop', '1', mode=None),
-        ])
-        mock_util.wait_for_removal.assert_has_calls([
-            mock.call(cset, retries=self.remove_retries)
-        ])
-
-    # test bcache shutdown with 'stop' sysfs write failure
     @mock.patch('curtin.block.clear_holders.block')
-    @mock.patch('curtin.block.wipe_volume')
-    @mock.patch('curtin.block.clear_holders.udev.udevadm_settle')
-    @mock.patch('curtin.block.clear_holders.get_bcache_sys_path')
-    @mock.patch('curtin.block.clear_holders.util')
-    @mock.patch('curtin.block.clear_holders.os')
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.get_bcache_using_dev')
-    def test_shutdown_bcache_stop_sysfs_write_fails(self, mock_get_bcache,
-                                                    mock_log, mock_os,
-                                                    mock_util,
-                                                    mock_get_bcache_block,
-                                                    mock_udevadm_settle,
-                                                    mock_wipe,
-                                                    mock_block):
-        """Test writes sysfs write failures pass if file not present"""
-        mock_block.sysfs_to_devpath.return_value = self.test_blockdev
-        mock_os.path.exists.side_effect = iter([
-                True,   # backing device exists
-                True,   # cset device not present (already removed)
-                False,  # backing device is removed with cset
-                False,  # bcache/stop sysfs is missing (already removed)
-        ])
-        cset = '/sys/fs/bcache/fake'
-        mock_get_bcache.return_value = cset
-        mock_get_bcache_block.return_value = self.test_syspath + '/bcache'
-        mock_os.path.join.side_effect = os.path.join
-
-        # make writes to sysfs fail
-        mock_util.write_file.side_effect = IOError(errno.ENOENT,
-                                                   "File not found")
-
+    def test_shutdown_bcache_no_device(self, m_block, m_os):
+        """ shutdown_bcache does nothing if target device is not present."""
+        m_os.path.exists.return_value = False
         clear_holders.shutdown_bcache(self.test_syspath)
-
-        self.assertEqual(4, len(mock_log.info.call_args_list))
-        self.assertEqual(3, len(mock_os.path.exists.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache.call_args_list))
-        self.assertEqual(1, len(mock_get_bcache_block.call_args_list))
-        self.assertEqual(1, len(mock_util.write_file.call_args_list))
-        self.assertEqual(1, len(mock_util.wait_for_removal.call_args_list))
-
-        mock_get_bcache.assert_called_with(self.test_syspath, strict=False)
-        mock_util.write_file.assert_has_calls([
-            mock.call(cset + '/stop', '1', mode=None),
-        ])
-        mock_util.wait_for_removal.assert_has_calls([
-            mock.call(cset, retries=self.remove_retries)
-        ])
+        m_block.wipe_volume.assert_not_called()
 
     @mock.patch('curtin.block.quick_zero')
     @mock.patch('curtin.block.clear_holders.LOG')
@@ -757,41 +537,6 @@ class TestClearHolders(CiTestCase):
         for tree, result in test_trees_and_results:
             self.assertEqual(clear_holders.format_holders_tree(tree), result)
 
-    @mock.patch('curtin.block.clear_holders.util.write_file')
-    def test_maybe_stop_bcache_device_raises_errors(self, m_write_file):
-        """Non-IO/OS exceptions are raised by maybe_stop_bcache_device."""
-        m_write_file.side_effect = ValueError('Crazy Value Error')
-        with self.assertRaises(ValueError) as cm:
-            clear_holders.maybe_stop_bcache_device('does/not/matter')
-        self.assertEqual('Crazy Value Error', str(cm.exception))
-        self.assertEqual(
-            mock.call('does/not/matter/stop', '1', mode=None),
-            m_write_file.call_args)
-
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.util.write_file')
-    def test_maybe_stop_bcache_device_handles_oserror(self, m_write_file,
-                                                      m_log):
-        """When OSError.NOENT is raised, log the condition and move on."""
-        m_write_file.side_effect = OSError(errno.ENOENT, 'Expected oserror')
-        clear_holders.maybe_stop_bcache_device('does/not/matter')
-        self.assertEqual(
-            'Error writing to bcache stop file %s, device removed: %s',
-            m_log.debug.call_args[0][0])
-        self.assertEqual('does/not/matter/stop', m_log.debug.call_args[0][1])
-
-    @mock.patch('curtin.block.clear_holders.LOG')
-    @mock.patch('curtin.block.clear_holders.util.write_file')
-    def test_maybe_stop_bcache_device_handles_ioerror(self, m_write_file,
-                                                      m_log):
-        """When IOError.NOENT is raised, log the condition and move on."""
-        m_write_file.side_effect = IOError(errno.ENOENT, 'Expected ioerror')
-        clear_holders.maybe_stop_bcache_device('does/not/matter')
-        self.assertEqual(
-            'Error writing to bcache stop file %s, device removed: %s',
-            m_log.debug.call_args[0][0])
-        self.assertEqual('does/not/matter/stop', m_log.debug.call_args[0][1])
-
     def test_get_holder_types(self):
         """test clear_holders.get_holder_types"""
         test_trees_and_results = [
diff --git a/tests/vmtests/test_bcache_ceph.py b/tests/vmtests/test_bcache_ceph.py
new file mode 100644
index 00000000..30820bd3
--- /dev/null
+++ b/tests/vmtests/test_bcache_ceph.py
@@ -0,0 +1,93 @@
+# This file is part of curtin. See LICENSE file for copyright and license info.
+
+from . import VMBaseClass, skip_if_flag
+from .releases import base_vm_classes as relbase
+
+import glob
+import textwrap
+
+
+class TestBcacheCeph(VMBaseClass):
+    arch_skip = [
+        "s390x",  # lp:1565029
+    ]
+    test_type = 'storage'
+    conf_file = "examples/tests/bcache-ceph-nvme.yaml"
+    nr_cpus = 2
+    uefi = True
+    dirty_disks = True
+    extra_disks = ['20G', '20G', '20G', '20G', '20G', '20G', '20G', '20G']
+    nvme_disks = ['20G', '20G']
+    extra_collect_scripts = [textwrap.dedent("""
+        cd OUTPUT_COLLECT_D
+        ls /sys/fs/bcache/ > bcache_ls
+        ls -al /dev/bcache/by-uuid/ > ls_al_dev_bcache_by_uuid
+        ls -al /dev/bcache/by-label/ > ls_al_dev_bcache_by_label
+        ls -al /sys/class/block/bcache* > ls_al_sys_block_bcache
+        for bcache in /sys/class/block/bcache*; do
+            for link in $(find ${bcache}/slaves -type l); do
+                kname=$(basename $(readlink $link))
+                outfile="bcache-super-show.$kname"
+                bcache-super-show /dev/${kname} > $outfile
+            done
+        done
+        exit 0
+        """)]
+
+    @skip_if_flag('expected_failure')
+    def test_bcache_output_files_exist(self):
+        self.output_files_exist([
+            "bcache-super-show.vda1",
+            "bcache-super-show.vdc",
+            "bcache-super-show.vdd",
+            "bcache-super-show.vde",
+            "bcache-super-show.vdf",
+            "bcache-super-show.vdh",
+            "bcache-super-show.nvme0n1p2",
+            "bcache-super-show.nvme1n1p2"])
+
+    @skip_if_flag('expected_failure')
+    def test_bcache_devices_cset_found(self):
+        sblocks = glob.glob("%s/bcache-super-show.*")
+        for superblock in sblocks:
+            bcache_cset_uuid = None
+            for line in self.load_collect_file(superblock).splitlines():
+                if line != "" and line.split()[0] == "cset.uuid":
+                    bcache_cset_uuid = line.split()[-1].rstrip()
+            self.assertIsNotNone(bcache_cset_uuid)
+            self.assertTrue(bcache_cset_uuid in
+                            self.load_collect_file("bcache_ls").splitlines())
+
+
+class TrustyTestBcacheCeph(relbase.trusty, TestBcacheCeph):
+    __test__ = False  # covered by test_raid5_bcache
+
+
+class TrustyHWEXTestBcacheCeph(relbase.trusty_hwe_x, TestBcacheCeph):
+    __test__ = False  # covered by test_raid5_bcache
+
+
+class XenialGATestBcacheCeph(relbase.xenial_ga, TestBcacheCeph):
+    __test__ = True
+
+
+class XenialHWETestBcacheCeph(relbase.xenial_hwe, TestBcacheCeph):
+    __test__ = True
+
+
+class XenialEdgeTestBcacheCeph(relbase.xenial_edge, TestBcacheCeph):
+    __test__ = True
+
+
+class BionicTestBcacheCeph(relbase.bionic, TestBcacheCeph):
+    __test__ = True
+
+
+class CosmicTestBcacheCeph(relbase.cosmic, TestBcacheCeph):
+    __test__ = True
+
+
+class DiscoTestBcacheCeph(relbase.disco, TestBcacheCeph):
+    __test__ = True
+
+# vi: ts=4 expandtab syntax=python