drbd: serialize sending of resync uuid with pending w_send_oos

To improve the latency of IO requests during bitmap exchange,
we recently allowed writes while waiting for the bitmap, sending "set
out-of-sync" information packets for any newly dirtied bits.

We have to make sure that the new resync-uuid does not overtake
these "set oos" packets. Once the resync-uuid is received, the
sync target starts the resync process, and expects the bitmap to
only be cleared, not re-set.

If we use this protocol extension, we queue the generation and sending
of the resync-uuid on the worker, which naturally serializes with all
previously queued packets.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 5a2d0ec72b34c738fc5e0ae71319f11ecf74e99e..ec06e744be42fdfd1d742f654c8a1f6d9a3089b2 100644 (file)
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1204,7 +1204,7 @@ extern int drbd_send(struct drbd_conf *mdev, struct socket *sock,
 extern int drbd_send_protocol(struct drbd_conf *mdev);
 extern int drbd_send_uuids(struct drbd_conf *mdev);
 extern int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev);
-extern int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val);
+extern int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev);
 extern int drbd_send_sizes(struct drbd_conf *mdev, int trigger_reply, enum dds_flags flags);
 extern int _drbd_send_state(struct drbd_conf *mdev);
 extern int drbd_send_state(struct drbd_conf *mdev);

diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 4da6f11cc82e65d929e3b00b76d4790daa0b38fb..2190064d59bd0d0be9485ad5eba85e410bd59025 100644 (file)
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1387,6 +1387,17 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
                spin_unlock_irq(&mdev->req_lock);
        }
 
+       /* Became sync source.  With protocol >= 96, we still need to send out
+        * the sync uuid now. Need to do that before any drbd_send_state, or
+        * the other side may go "paused sync" before receiving the sync uuids,
+        * which is unexpected. */
+       if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) &&
+           (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) &&
+           mdev->agreed_pro_version >= 96 && get_ldev(mdev)) {
+               drbd_gen_and_send_sync_uuid(mdev);
+               put_ldev(mdev);
+       }
+
        /* Do not change the order of the if above and the two below... */
        if (os.pdsk == D_DISKLESS && ns.pdsk > D_DISKLESS) {      /* attach on the peer */
                drbd_send_uuids(mdev);
@@ -1980,12 +1991,17 @@ int drbd_send_uuids_skip_initial_sync(struct drbd_conf *mdev)
        return _drbd_send_uuids(mdev, 8);
 }
 
-
-int drbd_send_sync_uuid(struct drbd_conf *mdev, u64 val)
+int drbd_gen_and_send_sync_uuid(struct drbd_conf *mdev)
 {
        struct p_rs_uuid p;
+       u64 uuid;
+
+       D_ASSERT(mdev->state.disk == D_UP_TO_DATE);
 
-       p.uuid = cpu_to_be64(val);
+       get_random_bytes(&uuid, sizeof(u64));
+       drbd_uuid_set(mdev, UI_BITMAP, uuid);
+       drbd_md_sync(mdev);
+       p.uuid = cpu_to_be64(uuid);
 
        return drbd_send_cmd(mdev, USE_DATA_SOCKET, P_SYNC_UUID,
                             (struct p_header80 *)&p, sizeof(p));

diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 2374454cdf17f437be884e46d0a38f243f80c683..3df37e65c118307d003a9ad24cc17403c0b73fa0 100644 (file)
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -1516,18 +1516,6 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
                return;
        }
 
-       if (side == C_SYNC_TARGET) {
-               mdev->bm_resync_fo = 0;
-       } else /* side == C_SYNC_SOURCE */ {
-               u64 uuid;
-
-               get_random_bytes(&uuid, sizeof(u64));
-               drbd_uuid_set(mdev, UI_BITMAP, uuid);
-               drbd_send_sync_uuid(mdev, uuid);
-
-               D_ASSERT(mdev->state.disk == D_UP_TO_DATE);
-       }
-
        write_lock_irq(&global_state_lock);
        ns = mdev->state;
 
@@ -1565,7 +1553,19 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
                _drbd_pause_after(mdev);
        }
        write_unlock_irq(&global_state_lock);
-       put_ldev(mdev);
+
+       if (side == C_SYNC_TARGET)
+               mdev->bm_resync_fo = 0;
+
+       /* Since protocol 96, we must serialize drbd_gen_and_send_sync_uuid
+        * with w_send_oos, or the sync target will get confused as to
+        * how much bits to resync.  We cannot do that always, because for an
+        * empty resync and protocol < 95, we need to do it here, as we call
+        * drbd_resync_finished from here in that case.
+        * We drbd_gen_and_send_sync_uuid here for protocol < 96,
+        * and from after_state_ch otherwise. */
+       if (side == C_SYNC_SOURCE && mdev->agreed_pro_version < 96)
+               drbd_gen_and_send_sync_uuid(mdev);
 
        if (r == SS_SUCCESS) {
                dev_info(DEV, "Began resync as %s (will sync %lu KB [%lu bits set]).\n",
@@ -1601,6 +1601,7 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
 
                drbd_md_sync(mdev);
        }
+       put_ldev(mdev);
        drbd_state_unlock(mdev);
 }