Update radix-tree.[ch] from upstream Linux to gain RCU awareness.

We still leave behind features we don't need such as tagged nodes.

Other changes:
 - Allow callers to define their own node alloc routines.
 - Only allocate per-node rcu_head when using the default RCU-safe
   alloc routines.
 - Keep our own radix_tree_destroy().

In future it may also be worth getting rid of the complex and
pointless special-casing of radix-tree height==0, in which a single
data item can be stored directly in radix_tree_root. It introduces a
whole lot of special cases and complicates RCU handling. If we get rid
of it we can reclaim the 'indirect pointer' tag in bit 0 of every slot
entry.

Signed-off-by: Keir Fraser <keir@xen.org>

xen/common/tmem.c

@@ -772,15 +772,12 @@ static NOINLINE void pgp_destroy(void *v)
     pgp_free(pgp,0);
 }
 
-FORWARD static rtn_t *rtn_alloc(void *arg);
-FORWARD static void rtn_free(rtn_t *rtn);
-
 static int pgp_add_to_obj(obj_t *obj, uint32_t index, pgp_t *pgp)
 {
     int ret;
 
     ASSERT_SPINLOCK(&obj->obj_spinlock);
-    ret = radix_tree_insert(&obj->tree_root, index, pgp, rtn_alloc, obj);
+    ret = radix_tree_insert(&obj->tree_root, index, pgp);
     if ( !ret )
         obj->pgp_count++;
     return ret;
@@ -795,7 +792,7 @@ static NOINLINE pgp_t *pgp_delete_from_obj(obj_t *obj, uint32_t index)
     ASSERT_SENTINEL(obj,OBJ);
     ASSERT(obj->pool != NULL);
     ASSERT_SENTINEL(obj->pool,POOL);
-    pgp = radix_tree_delete(&obj->tree_root, index, rtn_free);
+    pgp = radix_tree_delete(&obj->tree_root, index);
     if ( pgp != NULL )
         obj->pgp_count--;
     ASSERT(obj->pgp_count >= 0);
@@ -828,15 +825,12 @@ static NOINLINE rtn_t *rtn_alloc(void *arg)
 }
 
 /* called only indirectly from radix_tree_delete/destroy */
-static void rtn_free(rtn_t *rtn)
+static void rtn_free(rtn_t *rtn, void *arg)
 {
     pool_t *pool;
     objnode_t *objnode;
-    int i;
 
     ASSERT(rtn != NULL);
-    for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
-        ASSERT(rtn->slots[i] == NULL);
     objnode = container_of(rtn,objnode_t,rtn);
     ASSERT_SENTINEL(objnode,OBJNODE);
     INVERT_SENTINEL(objnode,OBJNODE);
@@ -943,7 +937,7 @@ static NOINLINE void obj_free(obj_t *obj, int no_rebalance)
     ASSERT(pool->client != NULL);
     ASSERT_WRITELOCK(&pool->pool_rwlock);
     if ( obj->tree_root.rnode != NULL ) /* may be a "stump" with no leaves */
-        radix_tree_destroy(&obj->tree_root, pgp_destroy, rtn_free);
+        radix_tree_destroy(&obj->tree_root, pgp_destroy);
     ASSERT((long)obj->objnode_count == 0);
     ASSERT(obj->tree_root.rnode == NULL);
     pool->obj_count--;
@@ -1003,7 +997,8 @@ static NOINLINE obj_t * obj_new(pool_t *pool, OID *oidp)
     if (pool->obj_count > pool->obj_count_max)
         pool->obj_count_max = pool->obj_count;
     atomic_inc_and_max(global_obj_count);
-    INIT_RADIX_TREE(&obj->tree_root,0);
+    radix_tree_init(&obj->tree_root);
+    radix_tree_set_alloc_callbacks(&obj->tree_root, rtn_alloc, rtn_free, obj);
     spin_lock_init(&obj->obj_spinlock);
     obj->pool = pool;
     obj->oid = *oidp;
@@ -1022,7 +1017,7 @@ static NOINLINE obj_t * obj_new(pool_t *pool, OID *oidp)
 static NOINLINE void obj_destroy(obj_t *obj, int no_rebalance)
 {
     ASSERT_WRITELOCK(&obj->pool->pool_rwlock);
-    radix_tree_destroy(&obj->tree_root, pgp_destroy, rtn_free);
+    radix_tree_destroy(&obj->tree_root, pgp_destroy);
     obj_free(obj,no_rebalance);
 }
 
@@ -2925,7 +2920,6 @@ static int __init init_tmem(void)
     if ( !tmh_enabled() )
         return 0;
 
-    radix_tree_init();
     if ( tmh_dedup_enabled() )
         for (i = 0; i < 256; i++ )
         {

xen/include/xen/lib.h

@@ -48,7 +48,8 @@ do {                                                            \
 #define SWAP(_a, _b) \
    do { typeof(_a) _t = (_a); (_a) = (_b); (_b) = _t; } while ( 0 )
 
-#define DIV_ROUND(x, y) (((x) + (y) / 2) / (y))
+#define DIV_ROUND(n, d) (((n) + (d) / 2) / (d))
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
 
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]) + __must_be_array(x))
 

xen/include/xen/radix-tree.h

@@ -1,7 +1,7 @@
 /*
  * Copyright (C) 2001 Momchil Velikov
  * Portions Copyright (C) 2001 Christoph Hellwig
- * Adapted for Xen by Dan Magenheimer, Oracle Corp.
+ * Copyright (C) 2006 Nick Piggin
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -20,59 +20,190 @@
 #ifndef _XEN_RADIX_TREE_H
 #define _XEN_RADIX_TREE_H
 
-/* root tags are stored in gfp_mask, shifted by __GFP_BITS_SHIFT */
-struct radix_tree_root {
-    unsigned int height;
-    struct radix_tree_node *rnode;
-};
+#include <xen/config.h>
+#include <xen/types.h>
+#include <xen/lib.h>
+#include <xen/rcupdate.h>
 
-#define RADIX_TREE_MAP_SHIFT 6
+/*
+ * An indirect pointer (root->rnode pointing to a radix_tree_node, rather
+ * than a data item) is signalled by the low bit set in the root->rnode
+ * pointer.
+ *
+ * In this case root->height is > 0, but the indirect pointer tests are
+ * needed for RCU lookups (because root->height is unreliable). The only
+ * time callers need worry about this is when doing a lookup_slot under
+ * RCU.
+ *
+ * Indirect pointer in fact is also used to tag the last pointer of a node
+ * when it is shrunk, before we rcu free the node. See shrink code for
+ * details.
+ */
+#define RADIX_TREE_INDIRECT_PTR	1
 
-#define RADIX_TREE_MAP_SIZE (1UL << RADIX_TREE_MAP_SHIFT)
-#define RADIX_TREE_MAP_MASK (RADIX_TREE_MAP_SIZE-1)
-
-#define RADIX_TREE_TAG_LONGS \
- ((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
-
-struct radix_tree_node {
-    unsigned int count;
-    void  *slots[RADIX_TREE_MAP_SIZE];
-};
-
-struct radix_tree_path {
-    struct radix_tree_node *node;
-    int offset;
-};
-
-#define RADIX_TREE_INDEX_BITS  (8 /* CHAR_BIT */ * sizeof(unsigned long))
-#define RADIX_TREE_MAX_PATH (RADIX_TREE_INDEX_BITS/RADIX_TREE_MAP_SHIFT + 2)
-
-
-#define RADIX_TREE_INIT(mask) {     \
- .height = 0,       \
- .rnode = NULL,       \
+static inline int radix_tree_is_indirect_ptr(void *ptr)
+{
+	return (int)((unsigned long)ptr & RADIX_TREE_INDIRECT_PTR);
 }
 
-#define RADIX_TREE(name, mask) \
- struct radix_tree_root name = RADIX_TREE_INIT(mask)
+/*
+ *** Radix tree structure definitions.
+ *** These are public to allow users to allocate instances of them.
+ *** However all fields are absolutely private.
+ */
 
-#define INIT_RADIX_TREE(root, mask)     \
-do {         \
- (root)->height = 0;      \
- (root)->rnode = NULL;      \
-} while (0)
+#define RADIX_TREE_MAP_SHIFT	6
+#define RADIX_TREE_MAP_SIZE	(1UL << RADIX_TREE_MAP_SHIFT)
+#define RADIX_TREE_MAP_MASK	(RADIX_TREE_MAP_SIZE-1)
 
-int radix_tree_insert(struct radix_tree_root *root, unsigned long index,
-                      void *item, struct radix_tree_node *(*node_alloc)(void *), void *arg);
+struct radix_tree_node {
+	unsigned int	height;		/* Height from the bottom */
+	unsigned int	count;
+	void __rcu	*slots[RADIX_TREE_MAP_SIZE];
+};
+
+typedef struct radix_tree_node *radix_tree_alloc_fn_t(void *);
+typedef void radix_tree_free_fn_t(struct radix_tree_node *, void *);
+
+struct radix_tree_root {
+	unsigned int		height;
+	struct radix_tree_node	__rcu *rnode;
+
+	/* Allow to specify custom node alloc/dealloc routines. */
+	radix_tree_alloc_fn_t *node_alloc;
+	radix_tree_free_fn_t *node_free;
+	void *node_alloc_free_arg;
+};
+
+/*
+ *** radix-tree API starts here **
+ */
+
+void radix_tree_init(struct radix_tree_root *root);
+void radix_tree_set_alloc_callbacks(
+	struct radix_tree_root *root,
+	radix_tree_alloc_fn_t *node_alloc,
+	radix_tree_free_fn_t *node_free,
+	void *node_alloc_free_arg);
+
+void radix_tree_destroy(
+	struct radix_tree_root *root,
+	void (*slot_free)(void *));
+
+/**
+ * Radix-tree synchronization
+ *
+ * The radix-tree API requires that users provide all synchronisation (with
+ * specific exceptions, noted below).
+ *
+ * Synchronization of access to the data items being stored in the tree, and
+ * management of their lifetimes must be completely managed by API users.
+ *
+ * For API usage, in general,
+ * - any function _modifying_ the tree (inserting or deleting items) must
+ *   exclude other modifications, and exclude any functions reading the tree.
+ * - any function _reading_ the tree (looking up items) must exclude
+ *   modifications to the tree, but may occur concurrently with other readers.
+ *
+ * The notable exceptions to this rule are the following functions:
+ * radix_tree_lookup
+ * radix_tree_lookup_slot
+ * radix_tree_gang_lookup
+ * radix_tree_gang_lookup_slot
+ *
+ * The first 7 functions are able to be called locklessly, using RCU. The
+ * caller must ensure calls to these functions are made within rcu_read_lock()
+ * regions. Other readers (lock-free or otherwise) and modifications may be
+ * running concurrently.
+ *
+ * It is still required that the caller manage the synchronization and lifetimes
+ * of the items. So if RCU lock-free lookups are used, typically this would mean
+ * that the items have their own locks, or are amenable to lock-free access; and
+ * that the items are freed by RCU (or only freed after having been deleted from
+ * the radix tree *and* a synchronize_rcu() grace period).
+ *
+ * (Note, rcu_assign_pointer and rcu_dereference are not needed to control
+ * access to data items when inserting into or looking up from the radix tree)
+ */
+
+/**
+ * radix_tree_deref_slot	- dereference a slot
+ * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
+ * Returns:	item that was stored in that slot with any direct pointer flag
+ *		removed.
+ *
+ * For use with radix_tree_lookup_slot().  Caller must hold tree at least read
+ * locked across slot lookup and dereference. Not required if write lock is
+ * held (ie. items cannot be concurrently inserted).
+ *
+ * radix_tree_deref_retry must be used to confirm validity of the pointer if
+ * only the read lock is held.
+ */
+static inline void *radix_tree_deref_slot(void **pslot)
+{
+	return rcu_dereference(*pslot);
+}
+
+/**
+ * radix_tree_deref_retry	- check radix_tree_deref_slot
+ * @arg:	pointer returned by radix_tree_deref_slot
+ * Returns:	0 if retry is not required, otherwise retry is required
+ *
+ * radix_tree_deref_retry must be used with radix_tree_deref_slot.
+ */
+static inline int radix_tree_deref_retry(void *arg)
+{
+	return unlikely((unsigned long)arg & RADIX_TREE_INDIRECT_PTR);
+}
+
+/**
+ * radix_tree_replace_slot	- replace item in a slot
+ * @pslot:	pointer to slot, returned by radix_tree_lookup_slot
+ * @item:	new item to store in the slot.
+ *
+ * For use with radix_tree_lookup_slot().  Caller must hold tree write locked
+ * across slot lookup and replacement.
+ */
+static inline void radix_tree_replace_slot(void **pslot, void *item)
+{
+	BUG_ON(radix_tree_is_indirect_ptr(item));
+	rcu_assign_pointer(*pslot, item);
+}
+
+
+/**
+ * radix_tree_{int_to_ptr,ptr_to_int}:
+ * 
+ * Allow storage of signed integers in radix-tree slots. We use an encoding
+ * in which the bottom two bits of the slot pointer are reserved (bit 0 for
+ * the indirect-pointer tag; bit 1 always set to prevent an in-use
+ * integer-valued slot from being NULL and thus mistakenly being reaped).
+ */
+static inline void *radix_tree_int_to_ptr(int val)
+{
+    ASSERT((val <= (LONG_MAX >> 2)) && (val >= (LONG_MIN >> 2)));
+    return (void *)(((long)val << 2) | 0x2ul);
+}
+
+static inline int radix_tree_ptr_to_int(void *ptr)
+{
+    ASSERT(((long)ptr & 0x3) == 0x2);
+    return (int)((long)ptr >> 2);
+}
+
+int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
 void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
 void **radix_tree_lookup_slot(struct radix_tree_root *, unsigned long);
-void radix_tree_destroy(struct radix_tree_root *root,
-                        void (*slot_free)(void *), void (*node_free)(struct radix_tree_node *));
-void *radix_tree_delete(struct radix_tree_root *root, unsigned long index,
-                        void(*node_free)(struct radix_tree_node *));
+void *radix_tree_delete(struct radix_tree_root *, unsigned long);
 unsigned int
 radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
-                       unsigned long first_index, unsigned int max_items);
-void radix_tree_init(void);
+			unsigned long first_index, unsigned int max_items);
+unsigned int
+radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results,
+			unsigned long first_index, unsigned int max_items);
+unsigned long radix_tree_next_hole(struct radix_tree_root *root,
+				unsigned long index, unsigned long max_scan);
+unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
+				unsigned long index, unsigned long max_scan);
 
 #endif /* _XEN_RADIX_TREE_H */

xen/include/xen/rcupdate.h

@@ -38,6 +38,8 @@
 #include <xen/cpumask.h>
 #include <xen/preempt.h>
 
+#define __rcu
+
 /**
  * struct rcu_head - callback structure for use with RCU
  * @next: next update requests in a list