xenforeignmemory: work around bug in older privcmd

Versions of linux privcmd prior to commit dc9eab6fd94d ("return -ENOTTY
for unimplemented IOCTLs") will return -EINVAL rather than the conventional
-ENOTTY for unimplemented codes. This breaks the error path in
libxenforeignmemory resource mapping, which only translates ENOTTY into
EOPNOTSUPP to inform callers of the need to use an alternative (legacy)
mechanism.

This patch adds a new 'unimplemented' [1] ioctl code into the local
privcmd header which is then used to probe for the appropriate errno to
translate in the resource mapping error path

[1] this is a code that has, so far, never been used in any version of
    privcmd and will be added to future versions of the header in the
    linux source, to make sure it stays unimplemented.

Signed-off-by: Paul Durrant <paul.durrant@citrix.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>

tools/include/xen-sys/Linux/privcmd.h

@@ -114,5 +114,7 @@ typedef struct privcmd_mmap_resource {
 	_IOC(_IOC_NONE, 'P', 6, sizeof(domid_t))
 #define IOCTL_PRIVCMD_MMAP_RESOURCE				\
 	_IOC(_IOC_NONE, 'P', 7, sizeof(privcmd_mmap_resource_t))
+#define IOCTL_PRIVCMD_UNIMPLEMENTED				\
+	_IOC(_IOC_NONE, 'P', 0xFF, 0)
 
 #endif /* __LINUX_PUBLIC_PRIVCMD_H__ */

tools/libs/foreignmemory/linux.c

@@ -53,6 +53,23 @@ int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem)
         return -1;
     }
 
+    /*
+     * Older versions of privcmd return -EINVAL for unimplemented ioctls
+     * so we need to probe for the errno to use rather than just using
+     * the conventional ENOTTY.
+     */
+    if ( ioctl(fd, IOCTL_PRIVCMD_UNIMPLEMENTED, NULL) >= 0 )
+    {
+        xtl_log(fmem->logger, XTL_ERROR, -1, "xenforeignmemory",
+                "privcmd ioctl should not be implemented");
+        return -1;
+    }
+    else
+    {
+        fmem->unimpl_errno = errno;
+        errno = 0;
+    }
+
     fmem->fd = fd;
     return 0;
 }
@@ -307,7 +324,7 @@ int osdep_xenforeignmemory_map_resource(
     {
         int saved_errno;
 
-        if ( errno != ENOTTY && errno != EOPNOTSUPP )
+        if ( errno != fmem->unimpl_errno && errno != EOPNOTSUPP )
             PERROR("ioctl failed");
         else
             errno = EOPNOTSUPP;

tools/libs/foreignmemory/private.h

@@ -23,6 +23,7 @@ struct xenforeignmemory_handle {
     unsigned flags;
     int fd;
     Xentoolcore__Active_Handle tc_ah;
+    int unimpl_errno;
 };
 
 int osdep_xenforeignmemory_open(xenforeignmemory_handle *fmem);