From: Jens Axboe <jens.axboe@oracle.com>
Date: Tue, 27 Mar 2007 08:16:21 +0000 (+0200)
Subject: Update rbtree code
X-Git-Tag: fio-1.15~29
X-Git-Url: https://git.kernel.dk/?p=fio.git;a=commitdiff_plain;h=44b5374f4d64a066643afbc91d7f3f03bb0b71f8;hp=64b18e125219430fbfe69e61642dbcaa1861434e

Update rbtree code

Grab from the later kernels, they embed the color in the pointer
to save some space.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>
---

diff --git a/rbtree.c b/rbtree.c
index cc4093aa..6ad800fd 100644
--- a/rbtree.c
+++ b/rbtree.c
@@ -25,60 +25,66 @@
 static void __rb_rotate_left(struct rb_node *node, struct rb_root *root)
 {
 	struct rb_node *right = node->rb_right;
+	struct rb_node *parent = rb_parent(node);
 
 	if ((node->rb_right = right->rb_left))
-		right->rb_left->rb_parent = node;
+		rb_set_parent(right->rb_left, node);
 	right->rb_left = node;
 
-	if ((right->rb_parent = node->rb_parent))
+	rb_set_parent(right, parent);
+
+	if (parent)
 	{
-		if (node == node->rb_parent->rb_left)
-			node->rb_parent->rb_left = right;
+		if (node == parent->rb_left)
+			parent->rb_left = right;
 		else
-			node->rb_parent->rb_right = right;
+			parent->rb_right = right;
 	}
 	else
 		root->rb_node = right;
-	node->rb_parent = right;
+	rb_set_parent(node, right);
 }
 
 static void __rb_rotate_right(struct rb_node *node, struct rb_root *root)
 {
 	struct rb_node *left = node->rb_left;
+	struct rb_node *parent = rb_parent(node);
 
 	if ((node->rb_left = left->rb_right))
-		left->rb_right->rb_parent = node;
+		rb_set_parent(left->rb_right, node);
 	left->rb_right = node;
 
-	if ((left->rb_parent = node->rb_parent))
+	rb_set_parent(left, parent);
+
+	if (parent)
 	{
-		if (node == node->rb_parent->rb_right)
-			node->rb_parent->rb_right = left;
+		if (node == parent->rb_right)
+			parent->rb_right = left;
 		else
-			node->rb_parent->rb_left = left;
+			parent->rb_left = left;
 	}
 	else
 		root->rb_node = left;
-	node->rb_parent = left;
+	rb_set_parent(node, left);
 }
 
 void rb_insert_color(struct rb_node *node, struct rb_root *root)
 {
 	struct rb_node *parent, *gparent;
 
-	while ((parent = node->rb_parent) && parent->rb_color == RB_RED)
+	while ((parent = rb_parent(node)) && rb_is_red(parent))
 	{
-		gparent = parent->rb_parent;
+		gparent = rb_parent(parent);
 
 		if (parent == gparent->rb_left)
 		{
 			{
 				register struct rb_node *uncle = gparent->rb_right;
-				if (uncle && uncle->rb_color == RB_RED)
+				if (uncle && rb_is_red(uncle))
 				{
-					uncle->rb_color = RB_BLACK;
-					parent->rb_color = RB_BLACK;
-					gparent->rb_color = RB_RED;
+					rb_set_black(uncle);
+					rb_set_black(parent);
+					rb_set_red(gparent);
 					node = gparent;
 					continue;
 				}
@@ -93,17 +99,17 @@ void rb_insert_color(struct rb_node *node, struct rb_root *root)
 				node = tmp;
 			}
 
-			parent->rb_color = RB_BLACK;
-			gparent->rb_color = RB_RED;
+			rb_set_black(parent);
+			rb_set_red(gparent);
 			__rb_rotate_right(gparent, root);
 		} else {
 			{
 				register struct rb_node *uncle = gparent->rb_left;
-				if (uncle && uncle->rb_color == RB_RED)
+				if (uncle && rb_is_red(uncle))
 				{
-					uncle->rb_color = RB_BLACK;
-					parent->rb_color = RB_BLACK;
-					gparent->rb_color = RB_RED;
+					rb_set_black(uncle);
+					rb_set_black(parent);
+					rb_set_red(gparent);
 					node = gparent;
 					continue;
 				}
@@ -118,13 +124,13 @@ void rb_insert_color(struct rb_node *node, struct rb_root *root)
 				node = tmp;
 			}
 
-			parent->rb_color = RB_BLACK;
-			gparent->rb_color = RB_RED;
+			rb_set_black(parent);
+			rb_set_red(gparent);
 			__rb_rotate_left(gparent, root);
 		}
 	}
 
-	root->rb_node->rb_color = RB_BLACK;
+	rb_set_black(root->rb_node);
 }
 
 static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
@@ -132,43 +138,40 @@ static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
 {
 	struct rb_node *other;
 
-	while ((!node || node->rb_color == RB_BLACK) && node != root->rb_node)
+	while ((!node || rb_is_black(node)) && node != root->rb_node)
 	{
 		if (parent->rb_left == node)
 		{
 			other = parent->rb_right;
-			if (other->rb_color == RB_RED)
+			if (rb_is_red(other))
 			{
-				other->rb_color = RB_BLACK;
-				parent->rb_color = RB_RED;
+				rb_set_black(other);
+				rb_set_red(parent);
 				__rb_rotate_left(parent, root);
 				other = parent->rb_right;
 			}
-			if ((!other->rb_left ||
-			     other->rb_left->rb_color == RB_BLACK)
-			    && (!other->rb_right ||
-				other->rb_right->rb_color == RB_BLACK))
+			if ((!other->rb_left || rb_is_black(other->rb_left)) &&
+			    (!other->rb_right || rb_is_black(other->rb_right)))
 			{
-				other->rb_color = RB_RED;
+				rb_set_red(other);
 				node = parent;
-				parent = node->rb_parent;
+				parent = rb_parent(node);
 			}
 			else
 			{
-				if (!other->rb_right ||
-				    other->rb_right->rb_color == RB_BLACK)
+				if (!other->rb_right || rb_is_black(other->rb_right))
 				{
-					register struct rb_node *o_left;
+					struct rb_node *o_left;
 					if ((o_left = other->rb_left))
-						o_left->rb_color = RB_BLACK;
-					other->rb_color = RB_RED;
+						rb_set_black(o_left);
+					rb_set_red(other);
 					__rb_rotate_right(other, root);
 					other = parent->rb_right;
 				}
-				other->rb_color = parent->rb_color;
-				parent->rb_color = RB_BLACK;
+				rb_set_color(other, rb_color(parent));
+				rb_set_black(parent);
 				if (other->rb_right)
-					other->rb_right->rb_color = RB_BLACK;
+					rb_set_black(other->rb_right);
 				__rb_rotate_left(parent, root);
 				node = root->rb_node;
 				break;
@@ -177,38 +180,35 @@ static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
 		else
 		{
 			other = parent->rb_left;
-			if (other->rb_color == RB_RED)
+			if (rb_is_red(other))
 			{
-				other->rb_color = RB_BLACK;
-				parent->rb_color = RB_RED;
+				rb_set_black(other);
+				rb_set_red(parent);
 				__rb_rotate_right(parent, root);
 				other = parent->rb_left;
 			}
-			if ((!other->rb_left ||
-			     other->rb_left->rb_color == RB_BLACK)
-			    && (!other->rb_right ||
-				other->rb_right->rb_color == RB_BLACK))
+			if ((!other->rb_left || rb_is_black(other->rb_left)) &&
+			    (!other->rb_right || rb_is_black(other->rb_right)))
 			{
-				other->rb_color = RB_RED;
+				rb_set_red(other);
 				node = parent;
-				parent = node->rb_parent;
+				parent = rb_parent(node);
 			}
 			else
 			{
-				if (!other->rb_left ||
-				    other->rb_left->rb_color == RB_BLACK)
+				if (!other->rb_left || rb_is_black(other->rb_left))
 				{
 					register struct rb_node *o_right;
 					if ((o_right = other->rb_right))
-						o_right->rb_color = RB_BLACK;
-					other->rb_color = RB_RED;
+						rb_set_black(o_right);
+					rb_set_red(other);
 					__rb_rotate_left(other, root);
 					other = parent->rb_left;
 				}
-				other->rb_color = parent->rb_color;
-				parent->rb_color = RB_BLACK;
+				rb_set_color(other, rb_color(parent));
+				rb_set_black(parent);
 				if (other->rb_left)
-					other->rb_left->rb_color = RB_BLACK;
+					rb_set_black(other->rb_left);
 				__rb_rotate_right(parent, root);
 				node = root->rb_node;
 				break;
@@ -216,7 +216,7 @@ static void __rb_erase_color(struct rb_node *node, struct rb_node *parent,
 		}
 	}
 	if (node)
-		node->rb_color = RB_BLACK;
+		rb_set_black(node);
 }
 
 void rb_erase(struct rb_node *node, struct rb_root *root)
@@ -236,48 +236,41 @@ void rb_erase(struct rb_node *node, struct rb_root *root)
 		while ((left = node->rb_left) != NULL)
 			node = left;
 		child = node->rb_right;
-		parent = node->rb_parent;
-		color = node->rb_color;
+		parent = rb_parent(node);
+		color = rb_color(node);
 
 		if (child)
-			child->rb_parent = parent;
-		if (parent)
-		{
-			if (parent->rb_left == node)
-				parent->rb_left = child;
-			else
-				parent->rb_right = child;
-		}
-		else
-			root->rb_node = child;
-
-		if (node->rb_parent == old)
+			rb_set_parent(child, parent);
+		if (parent == old) {
+			parent->rb_right = child;
 			parent = node;
-		node->rb_parent = old->rb_parent;
-		node->rb_color = old->rb_color;
+		} else
+			parent->rb_left = child;
+
+		node->rb_parent_color = old->rb_parent_color;
 		node->rb_right = old->rb_right;
 		node->rb_left = old->rb_left;
 
-		if (old->rb_parent)
+		if (rb_parent(old))
 		{
-			if (old->rb_parent->rb_left == old)
-				old->rb_parent->rb_left = node;
+			if (rb_parent(old)->rb_left == old)
+				rb_parent(old)->rb_left = node;
 			else
-				old->rb_parent->rb_right = node;
+				rb_parent(old)->rb_right = node;
 		} else
 			root->rb_node = node;
 
-		old->rb_left->rb_parent = node;
+		rb_set_parent(old->rb_left, node);
 		if (old->rb_right)
-			old->rb_right->rb_parent = node;
+			rb_set_parent(old->rb_right, node);
 		goto color;
 	}
 
-	parent = node->rb_parent;
-	color = node->rb_color;
+	parent = rb_parent(node);
+	color = rb_color(node);
 
 	if (child)
-		child->rb_parent = parent;
+		rb_set_parent(child, parent);
 	if (parent)
 	{
 		if (parent->rb_left == node)
@@ -322,6 +315,11 @@ struct rb_node *rb_last(struct rb_root *root)
 
 struct rb_node *rb_next(struct rb_node *node)
 {
+	struct rb_node *parent;
+
+	if (rb_parent(node) == node)
+		return NULL;
+
 	/* If we have a right-hand child, go down and then left as far
 	   as we can. */
 	if (node->rb_right) {
@@ -337,14 +335,19 @@ struct rb_node *rb_next(struct rb_node *node)
 	   ancestor is a right-hand child of its parent, keep going
 	   up. First time it's a left-hand child of its parent, said
 	   parent is our 'next' node. */
-	while (node->rb_parent && node == node->rb_parent->rb_right)
-		node = node->rb_parent;
+	while ((parent = rb_parent(node)) && node == parent->rb_right)
+		node = parent;
 
-	return node->rb_parent;
+	return parent;
 }
 
 struct rb_node *rb_prev(struct rb_node *node)
 {
+	struct rb_node *parent;
+
+	if (rb_parent(node) == node)
+		return NULL;
+
 	/* If we have a left-hand child, go down and then right as far
 	   as we can. */
 	if (node->rb_left) {
@@ -356,8 +359,31 @@ struct rb_node *rb_prev(struct rb_node *node)
 
 	/* No left-hand children. Go up till we find an ancestor which
 	   is a right-hand child of its parent */
-	while (node->rb_parent && node == node->rb_parent->rb_left)
-		node = node->rb_parent;
+	while ((parent = rb_parent(node)) && node == parent->rb_left)
+		node = parent;
+
+	return parent;
+}
+
+void rb_replace_node(struct rb_node *victim, struct rb_node *new,
+		     struct rb_root *root)
+{
+	struct rb_node *parent = rb_parent(victim);
+
+	/* Set the surrounding nodes to point to the replacement */
+	if (parent) {
+		if (victim == parent->rb_left)
+			parent->rb_left = new;
+		else
+			parent->rb_right = new;
+	} else {
+		root->rb_node = new;
+	}
+	if (victim->rb_left)
+		rb_set_parent(victim->rb_left, new);
+	if (victim->rb_right)
+		rb_set_parent(victim->rb_right, new);
 
-	return node->rb_parent;
+	/* Copy the pointers/colour from the victim to the replacement */
+	*new = *victim;
 }
diff --git a/rbtree.h b/rbtree.h
index 2cb9e37c..344bc349 100644
--- a/rbtree.h
+++ b/rbtree.h
@@ -94,37 +94,48 @@ static inline struct page * rb_insert_page_cache(struct inode * inode,
 #ifndef	_LINUX_RBTREE_H
 #define	_LINUX_RBTREE_H
 
-#include <stdlib.h>
+#include <linux/kernel.h>
+#include <linux/stddef.h>
 
 struct rb_node
 {
-	struct rb_node *rb_parent;
-	int rb_color;
+	unsigned long  rb_parent_color;
 #define	RB_RED		0
 #define	RB_BLACK	1
 	struct rb_node *rb_right;
 	struct rb_node *rb_left;
-};
+} __attribute__((aligned(sizeof(long))));
+    /* The alignment might seem pointless, but allegedly CRIS needs it */
 
 struct rb_root
 {
 	struct rb_node *rb_node;
 };
 
-#undef offsetof
-#ifdef __compiler_offsetof
-#define offsetof(TYPE,MEMBER) __compiler_offsetof(TYPE,MEMBER)
-#else
-#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
-#endif
 
-#define container_of(ptr, type, member) ({                      \
-	const typeof( ((type *)0)->member ) *__mptr = (ptr);    \
-	(type *)( (char *)__mptr - offsetof(type,member) );})
+#define rb_parent(r)   ((struct rb_node *)((r)->rb_parent_color & ~3))
+#define rb_color(r)   ((r)->rb_parent_color & 1)
+#define rb_is_red(r)   (!rb_color(r))
+#define rb_is_black(r) rb_color(r)
+#define rb_set_red(r)  do { (r)->rb_parent_color &= ~1; } while (0)
+#define rb_set_black(r)  do { (r)->rb_parent_color |= 1; } while (0)
+
+static inline void rb_set_parent(struct rb_node *rb, struct rb_node *p)
+{
+	rb->rb_parent_color = (rb->rb_parent_color & 3) | (unsigned long)p;
+}
+static inline void rb_set_color(struct rb_node *rb, int color)
+{
+	rb->rb_parent_color = (rb->rb_parent_color & ~1) | color;
+}
 
 #define RB_ROOT	(struct rb_root) { NULL, }
 #define	rb_entry(ptr, type, member) container_of(ptr, type, member)
 
+#define RB_EMPTY_ROOT(root)	((root)->rb_node == NULL)
+#define RB_EMPTY_NODE(node)	(rb_parent(node) == node)
+#define RB_CLEAR_NODE(node)	(rb_set_parent(node, node))
+
 extern void rb_insert_color(struct rb_node *, struct rb_root *);
 extern void rb_erase(struct rb_node *, struct rb_root *);
 
@@ -134,11 +145,14 @@ extern struct rb_node *rb_prev(struct rb_node *);
 extern struct rb_node *rb_first(struct rb_root *);
 extern struct rb_node *rb_last(struct rb_root *);
 
+/* Fast replacement of a single node without remove/rebalance/add/rebalance */
+extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, 
+			    struct rb_root *root);
+
 static inline void rb_link_node(struct rb_node * node, struct rb_node * parent,
 				struct rb_node ** rb_link)
 {
-	node->rb_parent = parent;
-	node->rb_color = RB_RED;
+	node->rb_parent_color = (unsigned long )parent;
 	node->rb_left = node->rb_right = NULL;
 
 	*rb_link = node;