libxl_pci: Fix guest shutdown with PCI PT attached

Before the problematic commit, libxl used to ignore error when
destroying (force == true) a passthrough device. If the DM failed to
detach the pci device within the allowed time, the timed out error
raised skip part of pci_remove_*, but also raise the error up to the
caller of libxl__device_pci_destroy_all, libxl__destroy_domid, and
thus the destruction of the domain fails.

When a *pci_destroy* function is called (so we have force=true), error
should mostly be ignored. If the DM didn't confirmed that the device
is removed, we will print a warning and keep going if force=true.
The patch reorder the functions so that pci_remove_timeout() calls
pci_remove_detatched() like it's done when DM calls are successful.

We also clean the QMP states and associated timeouts earlier, as soon
as they are not needed anymore.

Reported-by: Sander Eikelenboom <linux@eikelenboom.it>
Fixes: fae4880c45
Signed-off-by: Anthony PERARD <anthony.perard@citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
Tested-by: Sander Eikelenboom <linux@eikelenboom.it>
Release-acked-by: Juergen Gross <jgross@suse.com>

tools/libxl/libxl_pci.c

@@ -1778,12 +1778,12 @@ static void pci_remove_qmp_retry_timer_cb(libxl__egc *egc,
     libxl__ev_time *ev, const struct timeval *requested_abs, int rc);
 static void pci_remove_qmp_query_cb(libxl__egc *egc,
     libxl__ev_qmp *qmp, const libxl__json_object *response, int rc);
+static void pci_remove_timeout(libxl__egc *egc,
+    libxl__ev_time *ev, const struct timeval *requested_abs, int rc);
 static void pci_remove_detatched(libxl__egc *egc,
     pci_remove_state *prs, int rc);
 static void pci_remove_stubdom_done(libxl__egc *egc,
     libxl__ao_device *aodev);
-static void pci_remove_timeout(libxl__egc *egc,
-    libxl__ev_time *ev, const struct timeval *requested_abs, int rc);
 static void pci_remove_done(libxl__egc *egc,
     pci_remove_state *prs, int rc);
 
@@ -2044,6 +2044,25 @@ out:
     pci_remove_detatched(egc, prs, rc); /* must be last */
 }
 
+static void pci_remove_timeout(libxl__egc *egc, libxl__ev_time *ev,
+                               const struct timeval *requested_abs,
+                               int rc)
+{
+    EGC_GC;
+    pci_remove_state *prs = CONTAINER_OF(ev, *prs, timeout);
+
+    /* Convenience aliases */
+    libxl_device_pci *const pcidev = prs->pcidev;
+
+    LOGD(WARN, prs->domid, "timed out waiting for DM to remove "
+         PCI_PT_QDEV_ID, pcidev->bus, pcidev->dev, pcidev->func);
+
+    /* If we timed out, we might still want to keep destroying the device
+     * (when force==true), so let the next function decide what to do on
+     * error */
+    pci_remove_detatched(egc, prs, rc);
+}
+
 static void pci_remove_detatched(libxl__egc *egc,
                                  pci_remove_state *prs,
                                  int rc)
@@ -2057,6 +2076,11 @@ static void pci_remove_detatched(libxl__egc *egc,
     libxl_device_pci *const pcidev = prs->pcidev;
     libxl_domid domid = prs->domid;
 
+    /* Cleaning QMP states ASAP */
+    libxl__ev_qmp_dispose(gc, &prs->qmp);
+    libxl__ev_time_deregister(gc, &prs->timeout);
+    libxl__ev_time_deregister(gc, &prs->retry_timer);
+
     if (rc && !prs->force)
         goto out;
 
@@ -2104,15 +2128,6 @@ static void pci_remove_stubdom_done(libxl__egc *egc,
     pci_remove_done(egc, prs, 0);
 }
 
-static void pci_remove_timeout(libxl__egc *egc, libxl__ev_time *ev,
-                               const struct timeval *requested_abs,
-                               int rc)
-{
-    pci_remove_state *prs = CONTAINER_OF(ev, *prs, timeout);
-
-    pci_remove_done(egc, prs, rc);
-}
-
 static void pci_remove_done(libxl__egc *egc,
                             pci_remove_state *prs,
                             int rc)
@@ -2121,10 +2136,6 @@ static void pci_remove_done(libxl__egc *egc,
 
     if (rc) goto out;
 
-    libxl__ev_qmp_dispose(gc, &prs->qmp);
-    libxl__ev_time_deregister(gc, &prs->timeout);
-    libxl__ev_time_deregister(gc, &prs->retry_timer);
-
     libxl__device_pci_remove_xenstore(gc, prs->domid, prs->pcidev);
 out:
     device_pci_remove_common_next(egc, prs, rc);