From d9b394c7c2011a61a3485bd33f237066a05d1991 Mon Sep 17 00:00:00 2001 From: Sushmita Susheelendra Date: Thu, 16 Feb 2017 15:04:13 -0700 Subject: [PATCH] drm/msm: Fix drm_mm bottom_up search The bottom up search of drm_mm hole nodes is currently broken. It does not correctly look for the first hole (smallest address) that fits the allocation from the bottom up. Use an RB tree to record hole nodes with their address as key, so that the hole with the smallest address that satisfies the request can be looked up. Change-Id: Ie3020cb4e8b6db13a35d0fe5caaa8781cb6ced9f Signed-off-by: Sushmita Susheelendra --- drivers/gpu/drm/drm_mm.c | 108 +++++++++++++++++++++++++++--- drivers/gpu/drm/msm/msm_gem_vma.c | 2 +- include/drm/drm_mm.h | 3 + 3 files changed, 104 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c index 6b81035b51a1..6e4dd62d4ed9 100644 --- a/drivers/gpu/drm/drm_mm.c +++ b/drivers/gpu/drm/drm_mm.c @@ -47,6 +47,7 @@ #include #include #include +#include /** * DOC: Overview @@ -74,7 +75,8 @@ * allocations and avoiding too much fragmentation. This means free space * searches are O(num_holes). Given that all the fancy features drm_mm supports * something better would be fairly complex and since gfx thrashing is a fairly - * steep cliff not a real concern. Removing a node again is O(1). + * steep cliff not a real concern. Removing a node again is O(1). With the + * rbtree to track free holes, free hole search becomes O(log(num_holes)). * * drm_mm supports a few features: Alignment and range restrictions can be * supplied. Further more every &drm_mm_node has a color value (which is just an @@ -170,6 +172,32 @@ static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node, &drm_mm_interval_tree_augment); } +static void +rb_insert_hole_node(struct drm_mm_node *hole_node, struct drm_mm *mm) +{ + struct rb_node **new = &(mm->holes_tree.rb_node); + struct rb_node *parent = NULL; + struct drm_mm_node *cur; + + while (*new) { + parent = *new; + cur = rb_entry(parent, struct drm_mm_node, hole_node); + + if (__drm_mm_hole_node_start(hole_node) + < __drm_mm_hole_node_start(cur)) + new = &parent->rb_left; + else + new = &parent->rb_right; + } + rb_link_node(&hole_node->hole_node, parent, new); + rb_insert_color(&hole_node->hole_node, &mm->holes_tree); +} + +static void rb_erase_hole_node(struct drm_mm_node *hole_node, struct drm_mm *mm) +{ + rb_erase(&hole_node->hole_node, &mm->holes_tree); +} + static void drm_mm_insert_helper(struct drm_mm_node *hole_node, struct drm_mm_node *node, u64 size, unsigned alignment, @@ -209,6 +237,7 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node, if (adj_start == hole_start) { hole_node->hole_follows = 0; list_del(&hole_node->hole_stack); + rb_erase_hole_node(hole_node, mm); } node->start = adj_start; @@ -226,6 +255,7 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node, node->hole_follows = 0; if (__drm_mm_hole_node_start(node) < hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } } @@ -283,11 +313,13 @@ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node) if (node->start == hole_start) { hole->hole_follows = 0; list_del(&hole->hole_stack); + rb_erase_hole_node(hole, mm); } node->hole_follows = 0; if (end != hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } @@ -373,6 +405,7 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, if (adj_start == hole_start) { hole_node->hole_follows = 0; list_del(&hole_node->hole_stack); + rb_erase_hole_node(hole_node, mm); } node->start = adj_start; @@ -393,6 +426,7 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node, node->hole_follows = 0; if (__drm_mm_hole_node_start(node) < hole_end) { list_add(&node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(node, mm); node->hole_follows = 1; } } @@ -465,6 +499,7 @@ void drm_mm_remove_node(struct drm_mm_node *node) BUG_ON(__drm_mm_hole_node_start(node) == __drm_mm_hole_node_end(node)); list_del(&node->hole_stack); + rb_erase_hole_node(node, mm); } else BUG_ON(__drm_mm_hole_node_start(node) != __drm_mm_hole_node_end(node)); @@ -473,6 +508,7 @@ void drm_mm_remove_node(struct drm_mm_node *node) if (!prev_node->hole_follows) { prev_node->hole_follows = 1; list_add(&prev_node->hole_stack, &mm->hole_stack); + rb_insert_hole_node(prev_node, mm); } else list_move(&prev_node->hole_stack, &mm->hole_stack); @@ -499,6 +535,46 @@ static int check_free_hole(u64 start, u64 end, u64 size, unsigned alignment) return end >= start + size; } +static struct drm_mm_node *get_first_hole(const struct drm_mm *mm, + enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) { + struct rb_node *node = rb_first(&mm->holes_tree); + + return rb_entry(node, struct drm_mm_node, hole_node); + } else if (flags & DRM_MM_SEARCH_BELOW) { + return list_entry((mm)->hole_stack.prev, + struct drm_mm_node, hole_stack); + } else { + return list_entry((mm)->hole_stack.next, + struct drm_mm_node, hole_stack); + } +} + +static struct drm_mm_node *get_next_hole(struct drm_mm_node *entry, + enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) { + return rb_entry(rb_next(&entry->hole_node), + struct drm_mm_node, hole_node); + } else if (flags & DRM_MM_SEARCH_BELOW) { + return list_entry(entry->hole_stack.prev, + struct drm_mm_node, hole_stack); + } else { + return list_entry(entry->hole_stack.next, + struct drm_mm_node, hole_stack); + } +} + +static bool drm_mm_hole_traversal_condition(const struct drm_mm *mm, + struct drm_mm_node *entry, enum drm_mm_search_flags flags) +{ + if (flags & DRM_MM_SEARCH_BOTTOM_UP) + return entry ? 1 : 0; + else + return (&entry->hole_stack != &(mm)->hole_stack) ? 1 : 0; +} + static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, u64 size, unsigned alignment, @@ -516,9 +592,14 @@ static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm, best = NULL; best_size = ~0UL; - __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, - flags & DRM_MM_SEARCH_BELOW) { - u64 hole_size = adj_end - adj_start; + for (entry = get_first_hole(mm, flags); + drm_mm_hole_traversal_condition(mm, entry, flags); + entry = get_next_hole(entry, flags)) { + u64 hole_size; + + adj_start = drm_mm_hole_node_start(entry); + adj_end = drm_mm_hole_node_end(entry); + hole_size = adj_end - adj_start; if (mm->color_adjust) { mm->color_adjust(entry, color, &adj_start, &adj_end); @@ -560,9 +641,14 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_ best = NULL; best_size = ~0UL; - __drm_mm_for_each_hole(entry, mm, adj_start, adj_end, - flags & DRM_MM_SEARCH_BELOW) { - u64 hole_size = adj_end - adj_start; + for (entry = get_first_hole(mm, flags); + drm_mm_hole_traversal_condition(mm, entry, flags); + entry = get_next_hole(entry, flags)) { + u64 hole_size; + + adj_start = drm_mm_hole_node_start(entry); + adj_end = drm_mm_hole_node_end(entry); + hole_size = adj_end - adj_start; if (adj_start < start) adj_start = start; @@ -613,6 +699,11 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new) old->allocated = 0; new->allocated = 1; + + if (old->hole_follows) + rb_replace_node(&old->hole_node, &new->hole_node, + &old->mm->holes_tree); + } EXPORT_SYMBOL(drm_mm_replace_node); @@ -847,8 +938,9 @@ void drm_mm_init(struct drm_mm * mm, u64 start, u64 size) list_add_tail(&mm->head_node.hole_stack, &mm->hole_stack); mm->interval_tree = RB_ROOT; - mm->color_adjust = NULL; + mm->holes_tree = RB_ROOT; + rb_insert_hole_node(&mm->head_node, mm); } EXPORT_SYMBOL(drm_mm_init); diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index 95be430ea5d8..f399d24019e4 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -88,7 +88,7 @@ static int allocate_iova(struct msm_gem_address_space *aspace, return 0; } ret = drm_mm_insert_node(&aspace->mm, &vma->node, - size >> PAGE_SHIFT, 0, DRM_MM_SEARCH_DEFAULT); + size >> PAGE_SHIFT, 0, DRM_MM_SEARCH_BOTTOM_UP); spin_unlock(&aspace->lock); diff --git a/include/drm/drm_mm.h b/include/drm/drm_mm.h index debbbecbde1b..6649b6d8c437 100644 --- a/include/drm/drm_mm.h +++ b/include/drm/drm_mm.h @@ -49,6 +49,7 @@ enum drm_mm_search_flags { DRM_MM_SEARCH_DEFAULT = 0, DRM_MM_SEARCH_BEST = 1 << 0, DRM_MM_SEARCH_BELOW = 1 << 1, + DRM_MM_SEARCH_BOTTOM_UP = 1 << 2, }; enum drm_mm_allocator_flags { @@ -63,6 +64,7 @@ struct drm_mm_node { struct list_head node_list; struct list_head hole_stack; struct rb_node rb; + struct rb_node hole_node; unsigned hole_follows : 1; unsigned scanned_block : 1; unsigned scanned_prev_free : 1; @@ -84,6 +86,7 @@ struct drm_mm { struct drm_mm_node head_node; /* Keep an interval_tree for fast lookup of drm_mm_nodes by address. */ struct rb_root interval_tree; + struct rb_root holes_tree; unsigned int scan_check_range : 1; unsigned scan_alignment;