https://patchew.org/QEMU/20260719215455.96122-1-kirill@korins.ky/

Index: hw/usb/hcd-xhci.c
--- hw/usb/hcd-xhci.c.orig
+++ hw/usb/hcd-xhci.c
@@ -300,13 +300,19 @@ typedef struct XHCIEvRingSeg {
     uint32_t rsvd;
 } XHCIEvRingSeg;
 
+struct XHCIDeferredKick {
+    unsigned int index;
+    QTAILQ_ENTRY(XHCIDeferredKick) next;
+};
+
 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
                          unsigned int epid, unsigned int streamid);
 static void xhci_kick_epctx(XHCIEPContext *epctx, unsigned int streamid);
+static void xhci_process_commands(XHCIState *xhci);
 static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
                                 unsigned int epid);
 static void xhci_xfer_report(XHCITransfer *xfer);
-static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
+static bool xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
 static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);
 static USBEndpoint *xhci_epid_to_usbep(XHCIEPContext *epctx);
 
@@ -602,6 +608,304 @@ static inline int xhci_running(XHCIState *xhci)
     return !(xhci->usbsts & USBSTS_HCH);
 }
 
+static uint32_t xhci_pending_port_mask(const XHCIEvent *event)
+{
+    unsigned int portnr;
+
+    if (event->type != ER_PORT_STATUS_CHANGE) {
+        return 0;
+    }
+
+    portnr = event->ptr >> 24;
+    if (portnr < 1 || portnr > XHCI_MAXPORTS) {
+        return 0;
+    }
+    return 1U << (portnr - 1);
+}
+
+static bool xhci_pending_command_ring_stopped(const XHCIEvent *event)
+{
+    return event->type == ER_COMMAND_COMPLETE &&
+           event->ccode == CC_COMMAND_RING_STOPPED;
+}
+
+static void xhci_queue_pending_event(XHCIInterrupter *intr,
+                                     const XHCIEvent *event)
+{
+    XHCIPendingEvent *pending;
+    uint32_t port_mask;
+
+    if (event->type == ER_MFINDEX_WRAP) {
+        return;
+    }
+
+    port_mask = xhci_pending_port_mask(event);
+    if (port_mask && (intr->pending_port_events & port_mask)) {
+        return;
+    }
+    if (xhci_pending_command_ring_stopped(event)) {
+        if (intr->pending_command_ring_stopped) {
+            return;
+        }
+        intr->pending_command_ring_stopped = true;
+    }
+
+    pending = g_new(XHCIPendingEvent, 1);
+    pending->event = *event;
+    QTAILQ_INSERT_TAIL(&intr->pending_events, pending, next);
+    intr->pending_port_events |= port_mask;
+}
+
+static XHCIPendingEvent *xhci_pop_pending_event(XHCIInterrupter *intr)
+{
+    XHCIPendingEvent *pending = QTAILQ_FIRST(&intr->pending_events);
+
+    if (pending) {
+        intr->pending_port_events &=
+            ~xhci_pending_port_mask(&pending->event);
+        if (xhci_pending_command_ring_stopped(&pending->event)) {
+            intr->pending_command_ring_stopped = false;
+        }
+        QTAILQ_REMOVE(&intr->pending_events, pending, next);
+    }
+    return pending;
+}
+
+static void xhci_clear_pending_events(XHCIInterrupter *intr)
+{
+    XHCIPendingEvent *pending, *next_pending;
+
+    QTAILQ_FOREACH_SAFE(pending, &intr->pending_events, next, next_pending) {
+        QTAILQ_REMOVE(&intr->pending_events, pending, next);
+        g_free(pending);
+    }
+    intr->pending_port_events = 0;
+    intr->pending_command_ring_stopped = false;
+}
+
+static void xhci_set_event_ring_full(XHCIState *xhci,
+                                     XHCIInterrupter *intr, bool full)
+{
+    if (intr->er_full == full) {
+        return;
+    }
+
+    intr->er_full = full;
+    if (full) {
+        xhci->er_full_count++;
+    } else {
+        assert(xhci->er_full_count > 0);
+        xhci->er_full_count--;
+    }
+}
+
+static unsigned int xhci_endpoint_index(unsigned int slotid,
+                                        unsigned int epid)
+{
+    assert(slotid >= 1 && slotid <= XHCI_MAXSLOTS);
+    assert(epid >= 1 && epid <= 31);
+    return (slotid - 1) * 31 + epid - 1;
+}
+
+static bool xhci_deferred_ep_has_kicks(XHCIState *xhci,
+                                       unsigned int index)
+{
+    unsigned int word;
+
+    for (word = 0; word < XHCI_DEFERRED_KICK_WORDS; word++) {
+        if (xhci->deferred_kicks[index][word]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static void xhci_queue_deferred_ep(XHCIState *xhci, unsigned int index)
+{
+    XHCIDeferredKick *kick;
+
+    if (xhci->deferred_kick_eps[index]) {
+        return;
+    }
+
+    kick = g_new(XHCIDeferredKick, 1);
+    kick->index = index;
+    QTAILQ_INSERT_TAIL(&xhci->deferred_kick_queue, kick, next);
+    xhci->deferred_kick_eps[index] = kick;
+}
+
+static void xhci_defer_ep_kick(XHCIEPContext *epctx, unsigned int streamid)
+{
+    XHCIState *xhci = epctx->xhci;
+    unsigned int index;
+    unsigned int word;
+    uint64_t mask;
+
+    if (epctx->nr_pstreams) {
+        if (streamid == 0 || streamid >= epctx->nr_pstreams) {
+            return;
+        }
+    } else {
+        streamid = 0;
+    }
+
+    index = xhci_endpoint_index(epctx->slotid, epctx->epid);
+    word = streamid / 64;
+    mask = 1ULL << (streamid % 64);
+    if (!(xhci->deferred_kicks[index][word] & mask)) {
+        xhci->deferred_kicks[index][word] |= mask;
+        xhci->deferred_kick_count++;
+        xhci_queue_deferred_ep(xhci, index);
+    }
+}
+
+static void xhci_clear_ep_kicks(XHCIState *xhci, unsigned int slotid,
+                                unsigned int epid)
+{
+    unsigned int index = xhci_endpoint_index(slotid, epid);
+    XHCIDeferredKick *kick = xhci->deferred_kick_eps[index];
+    unsigned int word;
+
+    if (kick) {
+        QTAILQ_REMOVE(&xhci->deferred_kick_queue, kick, next);
+        xhci->deferred_kick_eps[index] = NULL;
+        g_free(kick);
+    }
+    for (word = 0; word < XHCI_DEFERRED_KICK_WORDS; word++) {
+        xhci->deferred_kick_count -=
+            ctpop64(xhci->deferred_kicks[index][word]);
+        xhci->deferred_kicks[index][word] = 0;
+    }
+}
+
+static void xhci_clear_deferred_kick_queue(XHCIState *xhci)
+{
+    XHCIDeferredKick *kick, *next_kick;
+
+    if (xhci->deferred_kick_bh) {
+        qemu_bh_cancel(xhci->deferred_kick_bh);
+    }
+    QTAILQ_FOREACH_SAFE(kick, &xhci->deferred_kick_queue,
+                        next, next_kick) {
+        QTAILQ_REMOVE(&xhci->deferred_kick_queue, kick, next);
+        g_free(kick);
+    }
+    memset(xhci->deferred_kick_eps, 0, sizeof(xhci->deferred_kick_eps));
+}
+
+static void xhci_clear_deferred_kicks(XHCIState *xhci)
+{
+    xhci_clear_deferred_kick_queue(xhci);
+    xhci->deferred_kick_count = 0;
+    memset(xhci->deferred_kicks, 0, sizeof(xhci->deferred_kicks));
+}
+
+static void xhci_rebuild_deferred_kick_queue(XHCIState *xhci)
+{
+    unsigned int index;
+
+    xhci_clear_deferred_kick_queue(xhci);
+    xhci->deferred_kick_count = 0;
+    for (index = 0; index < XHCI_MAX_ENDPOINT_CONTEXTS; index++) {
+        unsigned int word;
+
+        for (word = 0; word < XHCI_DEFERRED_KICK_WORDS; word++) {
+            xhci->deferred_kick_count +=
+                ctpop64(xhci->deferred_kicks[index][word]);
+        }
+        if (xhci_deferred_ep_has_kicks(xhci, index)) {
+            xhci_queue_deferred_ep(xhci, index);
+        }
+    }
+}
+
+static unsigned int xhci_first_deferred_stream(XHCIState *xhci,
+                                                unsigned int index)
+{
+    unsigned int word;
+
+    for (word = 0; word < XHCI_DEFERRED_KICK_WORDS; word++) {
+        uint64_t bits = xhci->deferred_kicks[index][word];
+
+        if (bits) {
+            return word * 64 + ctz64(bits);
+        }
+    }
+    return XHCI_MAX_STREAMS;
+}
+
+static void xhci_schedule_deferred_kicks(XHCIState *xhci)
+{
+    if (xhci->deferred_kick_bh && xhci_running(xhci) &&
+        xhci->er_full_count == 0 && xhci->deferred_kick_count) {
+        qemu_bh_schedule(xhci->deferred_kick_bh);
+    }
+}
+
+static void xhci_deferred_kick_bh(void *opaque)
+{
+    XHCIState *xhci = opaque;
+    XHCIDeferredKick *kick;
+    XHCIEPContext *epctx;
+    unsigned int index;
+    unsigned int slotid;
+    unsigned int epid;
+    unsigned int streamid;
+    unsigned int word;
+    uint64_t mask;
+
+    if (!xhci_running(xhci) || xhci->er_full_count ||
+        xhci->deferred_kick_count == 0) {
+        return;
+    }
+
+    kick = QTAILQ_FIRST(&xhci->deferred_kick_queue);
+    assert(kick != NULL);
+    index = kick->index;
+    QTAILQ_REMOVE(&xhci->deferred_kick_queue, kick, next);
+    xhci->deferred_kick_eps[index] = NULL;
+
+    streamid = xhci_first_deferred_stream(xhci, index);
+    assert(streamid < XHCI_MAX_STREAMS);
+    word = streamid / 64;
+    mask = 1ULL << (streamid % 64);
+    xhci->deferred_kicks[index][word] &= ~mask;
+    xhci->deferred_kick_count--;
+
+    if (xhci_deferred_ep_has_kicks(xhci, index)) {
+        QTAILQ_INSERT_TAIL(&xhci->deferred_kick_queue, kick, next);
+        xhci->deferred_kick_eps[index] = kick;
+    } else {
+        g_free(kick);
+    }
+
+    slotid = index / 31 + 1;
+    epid = index % 31 + 1;
+    epctx = xhci->slots[slotid - 1].eps[epid - 1];
+    if (epctx) {
+        xhci_kick_epctx(epctx, streamid);
+    }
+
+    /* The idle rearm yields instead of draining the queue in this poll. */
+    if (xhci_running(xhci) && xhci->er_full_count == 0 &&
+        xhci->deferred_kick_count) {
+        qemu_bh_schedule_idle(xhci->deferred_kick_bh);
+    }
+}
+
+static void xhci_resume_work(XHCIState *xhci)
+{
+    if (xhci->er_full_count) {
+        return;
+    }
+    if (xhci->crcr_low & CRCR_CRR) {
+        xhci_process_commands(xhci);
+    }
+    if (xhci->er_full_count == 0) {
+        xhci_schedule_deferred_kicks(xhci);
+    }
+}
+
 static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
     XHCIInterrupter *intr = &xhci->intr[v];
@@ -636,7 +940,7 @@ static void xhci_write_event(XHCIState *xhci, XHCIEven
     }
 }
 
-static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v)
+static bool xhci_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
     XHCIInterrupter *intr;
     dma_addr_t erdp;
@@ -649,10 +953,15 @@ static void xhci_event(XHCIState *xhci, XHCIEvent *eve
 
     if (v >= xhci->numintrs) {
         DPRINTF("intr nr out of range (%d >= %d)\n", v, xhci->numintrs);
-        return;
+        return false;
     }
     intr = &xhci->intr[v];
 
+    if (intr->er_full) {
+        xhci_queue_pending_event(intr, event);
+        return true;
+    }
+
     erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
     if (erdp < intr->er_start ||
         erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
@@ -660,25 +969,86 @@ static void xhci_event(XHCIState *xhci, XHCIEvent *eve
         DPRINTF("xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
                 v, intr->er_start, intr->er_size);
         xhci_die(xhci);
-        return;
+        return false;
     }
 
     dp_idx = (erdp - intr->er_start) / TRB_SIZE;
     assert(dp_idx < intr->er_size);
 
-    if ((intr->er_ep_idx + 2) % intr->er_size == dp_idx) {
+    if ((intr->er_ep_idx + 1) % intr->er_size == dp_idx) {
         DPRINTF("xhci: ER %d full, send ring full error\n", v);
         XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
+
+        xhci_queue_pending_event(intr, event);
+        intr->erdp_pending = false;
+        xhci_set_event_ring_full(xhci, intr, true);
         xhci_write_event(xhci, &full, v);
-    } else if ((intr->er_ep_idx + 1) % intr->er_size == dp_idx) {
-        DPRINTF("xhci: ER %d full, drop event\n", v);
-    } else {
-        xhci_write_event(xhci, event, v);
+        xhci_intr_raise(xhci, v);
+        return true;
     }
 
+    xhci_write_event(xhci, event, v);
     xhci_intr_raise(xhci, v);
+    return false;
 }
 
+static void xhci_event_ring_resume(XHCIState *xhci, int v)
+{
+    XHCIInterrupter *intr = &xhci->intr[v];
+    XHCIPendingEvent *pending;
+    dma_addr_t erdp;
+    unsigned int credits;
+    unsigned int dp_idx;
+    unsigned int free_entries;
+    bool posted = false;
+
+    if (!intr->er_full || !intr->erdp_pending || !xhci_running(xhci)) {
+        return;
+    }
+
+    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
+    if (erdp < intr->er_start ||
+        erdp >= intr->er_start + TRB_SIZE * intr->er_size) {
+        return;
+    }
+
+    intr->erdp_pending = false;
+    dp_idx = (erdp - intr->er_start) / TRB_SIZE;
+    free_entries = (dp_idx + intr->er_size - intr->er_ep_idx) %
+                   intr->er_size;
+    if (free_entries == 0) {
+        free_entries = intr->er_size;
+    }
+    credits = free_entries - 1;
+
+    /*
+     * ERDP cannot change during this MMIO operation, and one entry stays
+     * reserved for another Full Error.  Thus this snapshot can decrease
+     * at most er_size - 1 times and queued events cannot extend the loop.
+     */
+    while (credits &&
+           (pending = xhci_pop_pending_event(intr)) != NULL) {
+        credits--;
+        xhci_write_event(xhci, &pending->event, v);
+        g_free(pending);
+        posted = true;
+    }
+
+    if (!QTAILQ_EMPTY(&intr->pending_events)) {
+        XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
+
+        xhci_write_event(xhci, &full, v);
+        xhci_intr_raise(xhci, v);
+        return;
+    }
+
+    xhci_set_event_ring_full(xhci, intr, false);
+    if (posted) {
+        xhci_intr_raise(xhci, v);
+    }
+    xhci_resume_work(xhci);
+}
+
 static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,
                            dma_addr_t base)
 {
@@ -802,6 +1172,7 @@ static void xhci_er_reset(XHCIState *xhci, int v)
     XHCIInterrupter *intr = &xhci->intr[v];
     XHCIEvRingSeg seg;
     dma_addr_t erstba = xhci_addr64(intr->erstba_low, intr->erstba_high);
+    bool was_full = intr->er_full;
 
     if (intr->erstsz == 0 || erstba == 0) {
         /* disabled */
@@ -839,13 +1210,25 @@ static void xhci_er_reset(XHCIState *xhci, int v)
 
     DPRINTF("xhci: event ring[%d]:" DMA_ADDR_FMT " [%d]\n",
             v, intr->er_start, intr->er_size);
+
+    if (was_full) {
+        intr->erdp_pending = true;
+        xhci_event_ring_resume(xhci, v);
+    }
 }
 
 static void xhci_run(XHCIState *xhci)
 {
+    unsigned int i;
+
     trace_usb_xhci_run();
     xhci->usbsts &= ~USBSTS_HCH;
     xhci->mfindex_start = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    for (i = 0; i < xhci->numintrs; i++) {
+        xhci_event_ring_resume(xhci, i);
+    }
+    xhci_schedule_deferred_kicks(xhci);
 }
 
 static void xhci_stop(XHCIState *xhci)
@@ -853,6 +1236,7 @@ static void xhci_stop(XHCIState *xhci)
     trace_usb_xhci_stop();
     xhci->usbsts |= USBSTS_HCH;
     xhci->crcr_low &= ~CRCR_CRR;
+    qemu_bh_cancel(xhci->deferred_kick_bh);
 }
 
 static XHCIStreamContext *xhci_alloc_stream_contexts(unsigned count,
@@ -1287,6 +1671,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsig
         return CC_SUCCESS;
     }
 
+    xhci_clear_ep_kicks(xhci, slotid, epid);
     xhci_ep_nuke_xfers(xhci, slotid, epid, 0);
 
     epctx = slot->eps[epid-1];
@@ -1328,6 +1713,7 @@ static TRBCCode xhci_stop_ep(XHCIState *xhci, unsigned
         return CC_EP_NOT_ENABLED_ERROR;
     }
 
+    xhci_clear_ep_kicks(xhci, slotid, epid);
     if (xhci_ep_nuke_xfers(xhci, slotid, epid, CC_STOPPED) > 0) {
         DPRINTF("xhci: FIXME: endpoint stopped w/ xfers running, "
                 "data might be lost\n");
@@ -1373,6 +1759,7 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigne
         return CC_CONTEXT_STATE_ERROR;
     }
 
+    xhci_clear_ep_kicks(xhci, slotid, epid);
     if (xhci_ep_nuke_xfers(xhci, slotid, epid, 0) > 0) {
         DPRINTF("xhci: FIXME: endpoint reset w/ xfers running, "
                 "data might be lost\n");
@@ -1893,6 +2280,11 @@ static void xhci_kick_epctx(XHCIEPContext *epctx, unsi
     trace_usb_xhci_ep_kick(epctx->slotid, epctx->epid, streamid);
     assert(!epctx->kick_active);
 
+    if (xhci->er_full_count) {
+        xhci_defer_ep_kick(epctx, streamid);
+        return;
+    }
+
     /* If the device has been detached, but the guest has not noticed this
        yet the 2 above checks will succeed, but we must NOT continue */
     if (!xhci_slot_ok(xhci, epctx->slotid)) {
@@ -1943,6 +2335,11 @@ static void xhci_kick_epctx(XHCIEPContext *epctx, unsi
         epctx->retry = NULL;
     }
 
+    if (xhci->er_full_count) {
+        xhci_defer_ep_kick(epctx, streamid);
+        return;
+    }
+
     if (epctx->state == EP_HALTED) {
         DPRINTF("xhci: ep halted, not running schedule\n");
         return;
@@ -2015,6 +2412,11 @@ static void xhci_kick_epctx(XHCIEPContext *epctx, unsi
             xfer = NULL;
         }
 
+        if (xhci->er_full_count) {
+            xhci_defer_ep_kick(epctx, streamid);
+            break;
+        }
+
         if (epctx->state == EP_HALTED) {
             break;
         }
@@ -2490,6 +2892,9 @@ static void xhci_process_commands(XHCIState *xhci)
     }
 
     xhci->crcr_low |= CRCR_CRR;
+    if (xhci->er_full_count) {
+        return;
+    }
 
     while ((type = xhci_ring_fetch(xhci, &xhci->cmd_ring, &trb, &addr))) {
         event.ptr = addr;
@@ -2600,7 +3005,9 @@ static void xhci_process_commands(XHCIState *xhci)
             break;
         }
         event.slotid = slotid;
-        xhci_event(xhci, &event, 0);
+        if (xhci_event(xhci, &event, 0)) {
+            return;
+        }
 
         if (count++ > COMMAND_LIMIT) {
             trace_usb_xhci_enforced_limit("commands");
@@ -2716,6 +3123,8 @@ static void xhci_reset(DeviceState *dev)
     xhci->dcbaap_low = 0;
     xhci->dcbaap_high = 0;
     xhci->config = 0;
+    xhci->er_full_count = 0;
+    xhci_clear_deferred_kicks(xhci);
 
     for (i = 0; i < xhci->numslots; i++) {
         xhci_disable_slot(xhci, i+1);
@@ -2736,6 +3145,11 @@ static void xhci_reset(DeviceState *dev)
 
         xhci->intr[i].er_ep_idx = 0;
         xhci->intr[i].er_pcs = 1;
+        xhci->intr[i].er_full = false;
+        xhci->intr[i].erdp_pending = false;
+        xhci_clear_pending_events(&xhci->intr[i]);
+        xhci->intr[i].pending_port_events = 0;
+        xhci->intr[i].er_full_unused = false;
         xhci->intr[i].ev_buffer_put = 0;
         xhci->intr[i].ev_buffer_get = 0;
     }
@@ -3139,6 +3553,11 @@ static void xhci_runtime_write(void *ptr, hwaddr reg,
             intr->erdp_low &= ~ERDP_EHB;
         }
         intr->erdp_low = (val & ~ERDP_EHB) | (intr->erdp_low & ERDP_EHB);
+        if (size == 8) {
+            intr->erdp_high = val >> 32;
+        }
+        intr->erdp_pending = true;
+        xhci_event_ring_resume(xhci, v);
         if (val & ERDP_EHB) {
             dma_addr_t erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
             unsigned int dp_idx = (erdp - intr->er_start) / TRB_SIZE;
@@ -3151,6 +3570,7 @@ static void xhci_runtime_write(void *ptr, hwaddr reg,
         break;
     case 0x1c: /* ERDP high */
         intr->erdp_high = val;
+        xhci_event_ring_resume(xhci, v);
         break;
     default:
         trace_usb_xhci_unimplemented("oper write", reg);
@@ -3425,6 +3845,15 @@ static void usb_xhci_realize(DeviceState *dev, Error *
     if (xhci->numslots < 1) {
         xhci->numslots = 1;
     }
+
+    QTAILQ_INIT(&xhci->deferred_kick_queue);
+    xhci->deferred_kick_bh = qemu_bh_new_guarded(
+        xhci_deferred_kick_bh, xhci, &dev->mem_reentrancy_guard);
+
+    for (i = 0; i < xhci->numintrs; i++) {
+        QTAILQ_INIT(&xhci->intr[i].pending_events);
+    }
+
     if (xhci_get_flag(xhci, XHCI_FLAG_ENABLE_STREAMS)) {
         xhci->max_pstreams_mask = 7; /* == 256 primary streams */
     } else {
@@ -3470,6 +3899,14 @@ static void usb_xhci_unrealize(DeviceState *dev)
         xhci_disable_slot(xhci, i + 1);
     }
 
+    for (i = 0; i < xhci->numintrs; i++) {
+        xhci_clear_pending_events(&xhci->intr[i]);
+    }
+
+    xhci_clear_deferred_kicks(xhci);
+    qemu_bh_delete(xhci->deferred_kick_bh);
+    xhci->deferred_kick_bh = NULL;
+
     if (xhci->mfwrap_timer) {
         timer_free(xhci->mfwrap_timer);
         xhci->mfwrap_timer = NULL;
@@ -3488,17 +3925,70 @@ static void usb_xhci_unrealize(DeviceState *dev)
     usb_bus_release(&xhci->bus);
 }
 
+static int usb_xhci_pre_save(void *opaque)
+{
+    XHCIState *xhci = opaque;
+    unsigned int slotid;
+
+    if (xhci->er_full_count == 0) {
+        return 0;
+    }
+
+    for (slotid = 1; slotid <= xhci->numslots; slotid++) {
+        unsigned int epid;
+
+        for (epid = 1; epid <= 31; epid++) {
+            XHCIEPContext *epctx = xhci->slots[slotid - 1].eps[epid - 1];
+            XHCITransfer *xfer;
+
+            if (!epctx) {
+                continue;
+            }
+            QTAILQ_FOREACH(xfer, &epctx->transfers, next) {
+                xhci_defer_ep_kick(epctx, xfer->streamid);
+            }
+        }
+    }
+    return 0;
+}
+
 static int usb_xhci_post_load(void *opaque, int version_id)
 {
     XHCIState *xhci = opaque;
+    XHCIPendingEvent *pending;
     XHCISlot *slot;
     XHCIEPContext *epctx;
     dma_addr_t dcbaap, pctx;
     uint32_t slot_ctx[4];
     uint32_t ep_ctx[5];
-    int slotid, epid, state;
+    int slotid, epid, state, i;
     uint64_t addr;
 
+    xhci->er_full_count = 0;
+    for (i = 0; i < xhci->numintrs; i++) {
+        XHCIInterrupter *intr = &xhci->intr[i];
+
+        intr->pending_port_events = 0;
+        intr->pending_command_ring_stopped = false;
+        if (!intr->er_full && !QTAILQ_EMPTY(&intr->pending_events)) {
+            return -EINVAL;
+        }
+        if (intr->erdp_pending && !intr->er_full) {
+            return -EINVAL;
+        }
+        if (intr->er_full) {
+            xhci->er_full_count++;
+        }
+        QTAILQ_FOREACH(pending, &intr->pending_events, next) {
+            intr->pending_port_events |=
+                xhci_pending_port_mask(&pending->event);
+            intr->pending_command_ring_stopped |=
+                xhci_pending_command_ring_stopped(&pending->event);
+        }
+    }
+
+    xhci_rebuild_deferred_kick_queue(xhci);
+
     dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
 
     for (slotid = 1; slotid <= xhci->numslots; slotid++) {
@@ -3531,11 +4021,19 @@ static int usb_xhci_post_load(void *opaque, int versio
             xhci_init_epctx(epctx, pctx, ep_ctx);
             epctx->state = state;
             if (state == EP_RUNNING) {
-                /* kick endpoint after vmload is finished */
-                timer_mod(epctx->kick_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+                if (xhci->er_full_count && !epctx->nr_pstreams) {
+                    xhci_defer_ep_kick(epctx, 0);
+                } else if (xhci->er_full_count == 0 &&
+                           !xhci_deferred_ep_has_kicks(
+                               xhci, xhci_endpoint_index(slotid, epid))) {
+                    /* kick endpoint after vmload is finished */
+                    timer_mod(epctx->kick_timer,
+                              qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+                }
             }
         }
     }
+    xhci_schedule_deferred_kicks(xhci);
     return 0;
 }
 
@@ -3583,6 +4081,38 @@ static const VMStateDescription vmstate_xhci_event = {
     }
 };
 
+static const VMStateDescription vmstate_xhci_pending_event = {
+    .name = "xhci-pending-event",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (const VMStateField[]) {
+        VMSTATE_STRUCT(event, XHCIPendingEvent, 0,
+                       vmstate_xhci_event, XHCIEvent),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool xhci_event_ring_full_needed(void *opaque)
+{
+    XHCIInterrupter *intr = opaque;
+
+    return intr->er_full || !QTAILQ_EMPTY(&intr->pending_events);
+}
+
+static const VMStateDescription vmstate_xhci_event_ring_full = {
+    .name = "xhci-intr/event-ring-full",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = xhci_event_ring_full_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_BOOL(er_full, XHCIInterrupter),
+        VMSTATE_BOOL(erdp_pending, XHCIInterrupter),
+        VMSTATE_QTAILQ_V(pending_events, XHCIInterrupter, 1,
+                         vmstate_xhci_pending_event, XHCIPendingEvent, next),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool xhci_er_full(void *opaque, int version_id)
 {
     return false;
@@ -3617,12 +4147,37 @@ static const VMStateDescription vmstate_xhci_intr = {
                                   vmstate_xhci_event, XHCIEvent),
 
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * const []) {
+        &vmstate_xhci_event_ring_full,
+        NULL
     }
 };
 
+static bool xhci_deferred_kicks_needed(void *opaque)
+{
+    XHCIState *xhci = opaque;
+
+    return xhci->deferred_kick_count != 0;
+}
+
+static const VMStateDescription vmstate_xhci_deferred_kicks = {
+    .name = "xhci-core/deferred-kicks",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = xhci_deferred_kicks_needed,
+    .fields = (const VMStateField[]) {
+        VMSTATE_UINT64_2DARRAY(deferred_kicks, XHCIState,
+                               XHCI_MAX_ENDPOINT_CONTEXTS,
+                               XHCI_DEFERRED_KICK_WORDS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_xhci = {
     .name = "xhci-core",
     .version_id = 1,
+    .pre_save = usb_xhci_pre_save,
     .post_load = usb_xhci_post_load,
     .fields = (const VMStateField[]) {
         VMSTATE_STRUCT_VARRAY_UINT32(ports, XHCIState, numports, 1,
@@ -3648,6 +4203,10 @@ const VMStateDescription vmstate_xhci = {
         VMSTATE_STRUCT(cmd_ring, XHCIState, 1, vmstate_xhci_ring, XHCIRing),
 
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * const []) {
+        &vmstate_xhci_deferred_kicks,
+        NULL
     }
 };
 
