From bb4a340e075b7897ece109686bfa177f8518d2db Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:18:37 -0700 Subject: mm: rename anon_vma_lock to vma_lock_anon_vma Rename anon_vma_lock to vma_lock_anon_vma. This matches the naming style used in page_lock_anon_vma and will come in really handy further down in this patch series. Signed-off-by: Rik van Riel Acked-by: Mel Gorman Acked-by: KAMEZAWA Hiroyuki Tested-by: Larry Woodman Acked-by: Larry Woodman Reviewed-by: Minchan Kim Acked-by: Linus Torvalds Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 77216742c17..80cd162a8aa 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -99,14 +99,14 @@ static inline struct anon_vma *page_anon_vma(struct page *page) return page_rmapping(page); } -static inline void anon_vma_lock(struct vm_area_struct *vma) +static inline void vma_lock_anon_vma(struct vm_area_struct *vma) { struct anon_vma *anon_vma = vma->anon_vma; if (anon_vma) spin_lock(&anon_vma->lock); } -static inline void anon_vma_unlock(struct vm_area_struct *vma) +static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) { struct anon_vma *anon_vma = vma->anon_vma; if (anon_vma) -- cgit v1.2.3 From cba48b98f2348c814316c4b4f411a07a0e4a2bf9 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:18:38 -0700 Subject: mm: change direct call of spin_lock(anon_vma->lock) to inline function Subsitute a direct call of spin_lock(anon_vma->lock) with an inline function doing exactly the same. This makes it easier to do the substitution to the root anon_vma lock in a following patch. We will deal with the handful of special locks (nested, dec_and_lock, etc) separately. Signed-off-by: Rik van Riel Acked-by: Mel Gorman Acked-by: KAMEZAWA Hiroyuki Tested-by: Larry Woodman Acked-by: Larry Woodman Reviewed-by: Minchan Kim Acked-by: Linus Torvalds Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 80cd162a8aa..5f981be6141 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -113,6 +113,16 @@ static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) spin_unlock(&anon_vma->lock); } +static inline void anon_vma_lock(struct anon_vma *anon_vma) +{ + spin_lock(&anon_vma->lock); +} + +static inline void anon_vma_unlock(struct anon_vma *anon_vma) +{ + spin_unlock(&anon_vma->lock); +} + /* * anon_vma helper functions. */ -- cgit v1.2.3 From 5c341ee1dfc8fe69d66b1c8b19e463c6d7201ae1 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:18:39 -0700 Subject: mm: track the root (oldest) anon_vma Track the root (oldest) anon_vma in each anon_vma tree. Because we only take the lock on the root anon_vma, we cannot use the lock on higher-up anon_vmas to lock anything. This makes it impossible to do an indirect lookup of the root anon_vma, since the data structures could go away from under us. However, a direct pointer is safe because the root anon_vma is always the last one that gets freed on munmap or exit, by virtue of the same_vma list order and unlink_anon_vmas walking the list forward. [akpm@linux-foundation.org: fix typo] Signed-off-by: Rik van Riel Acked-by: Mel Gorman Acked-by: KAMEZAWA Hiroyuki Tested-by: Larry Woodman Acked-by: Larry Woodman Reviewed-by: Minchan Kim Acked-by: Linus Torvalds Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 5f981be6141..41fa6ddc621 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -26,6 +26,7 @@ */ struct anon_vma { spinlock_t lock; /* Serialize access to vma list */ + struct anon_vma *root; /* Root of this anon_vma tree */ #if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION) /* -- cgit v1.2.3 From 012f18004da33ba672e3c60838cc4898126174d3 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:18:40 -0700 Subject: mm: always lock the root (oldest) anon_vma Always (and only) lock the root (oldest) anon_vma whenever we do something in an anon_vma. The recently introduced anon_vma scalability is due to the rmap code scanning only the VMAs that need to be scanned. Many common operations still took the anon_vma lock on the root anon_vma, so always taking that lock is not expected to introduce any scalability issues. However, always taking the same lock does mean we only need to take one lock, which means rmap_walk on pages from any anon_vma in the vma is excluded from occurring during an munmap, expand_stack or other operation that needs to exclude rmap_walk and similar functions. Also add the proper locking to vma_adjust. Signed-off-by: Rik van Riel Tested-by: Larry Woodman Acked-by: Larry Woodman Reviewed-by: Minchan Kim Reviewed-by: KAMEZAWA Hiroyuki Acked-by: Mel Gorman Acked-by: Linus Torvalds Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 41fa6ddc621..af43cb9a050 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -104,24 +104,24 @@ static inline void vma_lock_anon_vma(struct vm_area_struct *vma) { struct anon_vma *anon_vma = vma->anon_vma; if (anon_vma) - spin_lock(&anon_vma->lock); + spin_lock(&anon_vma->root->lock); } static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) { struct anon_vma *anon_vma = vma->anon_vma; if (anon_vma) - spin_unlock(&anon_vma->lock); + spin_unlock(&anon_vma->root->lock); } static inline void anon_vma_lock(struct anon_vma *anon_vma) { - spin_lock(&anon_vma->lock); + spin_lock(&anon_vma->root->lock); } static inline void anon_vma_unlock(struct anon_vma *anon_vma) { - spin_unlock(&anon_vma->lock); + spin_unlock(&anon_vma->root->lock); } /* -- cgit v1.2.3 From 76545066c8521f3e32c849744744842b4df25b79 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:18:41 -0700 Subject: mm: extend KSM refcounts to the anon_vma root KSM reference counts can cause an anon_vma to exist after the processe it belongs to have already exited. Because the anon_vma lock now lives in the root anon_vma, we need to ensure that the root anon_vma stays around until after all the "child" anon_vmas have been freed. The obvious way to do this is to have a "child" anon_vma take a reference to the root in anon_vma_fork. When the anon_vma is freed at munmap or process exit, we drop the refcount in anon_vma_unlink and possibly free the root anon_vma. The KSM anon_vma reference count function also needs to be modified to deal with the possibility of freeing 2 levels of anon_vma. The easiest way to do this is to break out the KSM magic and make it generic. When compiling without CONFIG_KSM, this code is compiled out. Signed-off-by: Rik van Riel Tested-by: Larry Woodman Acked-by: Larry Woodman Reviewed-by: Minchan Kim Cc: KAMEZAWA Hiroyuki Acked-by: Mel Gorman Acked-by: Linus Torvalds Tested-by: Dave Young Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index af43cb9a050..dc9b3c0bf5d 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -81,6 +81,13 @@ static inline int anonvma_external_refcount(struct anon_vma *anon_vma) { return atomic_read(&anon_vma->external_refcount); } + +static inline void get_anon_vma(struct anon_vma *anon_vma) +{ + atomic_inc(&anon_vma->external_refcount); +} + +void drop_anon_vma(struct anon_vma *); #else static inline void anonvma_external_refcount_init(struct anon_vma *anon_vma) { @@ -90,6 +97,14 @@ static inline int anonvma_external_refcount(struct anon_vma *anon_vma) { return 0; } + +static inline void get_anon_vma(struct anon_vma *anon_vma) +{ +} + +static inline void drop_anon_vma(struct anon_vma *anon_vma) +{ +} #endif /* CONFIG_KSM */ static inline struct anon_vma *page_anon_vma(struct page *page) -- cgit v1.2.3 From ad8c2ee801ad7a52d919b478d9b2c7b39a72d295 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Mon, 9 Aug 2010 17:19:48 -0700 Subject: rmap: add exclusive page to private anon_vma on swapin On swapin it is fairly common for a page to be owned exclusively by one process. In that case we want to add the page to the anon_vma of that process's VMA, instead of to the root anon_vma. This will reduce the amount of rmap searching that the swapout code needs to do. Signed-off-by: Rik van Riel Cc: Andrea Arcangeli Cc: KOSAKI Motohiro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/rmap.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index dc9b3c0bf5d..d6661de56f3 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -162,6 +162,8 @@ static inline void anon_vma_merge(struct vm_area_struct *vma, */ void page_move_anon_rmap(struct page *, struct vm_area_struct *, unsigned long); void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long); +void do_page_add_anon_rmap(struct page *, struct vm_area_struct *, + unsigned long, int); void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long); void page_add_file_rmap(struct page *); void page_remove_rmap(struct page *); -- cgit v1.2.3 From 0fe6e20b9c4c53b3e97096ee73a0857f60aad43f Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Fri, 28 May 2010 09:29:16 +0900 Subject: hugetlb, rmap: add reverse mapping for hugepage This patch adds reverse mapping feature for hugepage by introducing mapcount for shared/private-mapped hugepage and anon_vma for private-mapped hugepage. While hugepage is not currently swappable, reverse mapping can be useful for memory error handler. Without this patch, memory error handler cannot identify processes using the bad hugepage nor unmap it from them. That is: - for shared hugepage: we can collect processes using a hugepage through pagecache, but can not unmap the hugepage because of the lack of mapcount. - for privately mapped hugepage: we can neither collect processes nor unmap the hugepage. This patch solves these problems. This patch include the bug fix given by commit 23be7468e8, so reverts it. Dependency: "hugetlb: move definition of is_vm_hugetlb_page() to hugepage_inline.h" ChangeLog since May 24. - create hugetlb_inline.h and move is_vm_hugetlb_index() in it. - move functions setting up anon_vma for hugepage into mm/rmap.c. ChangeLog since May 13. - rebased to 2.6.34 - fix logic error (in case that private mapping and shared mapping coexist) - move is_vm_hugetlb_page() into include/linux/mm.h to use this function from linear_page_index() - define and use linear_hugepage_index() instead of compound_order() - use page_move_anon_rmap() in hugetlb_cow() - copy exclusive switch of __set_page_anon_rmap() into hugepage counterpart. - revert commit 24be7468 completely Signed-off-by: Naoya Horiguchi Cc: Andi Kleen Cc: Andrew Morton Cc: Mel Gorman Cc: Andrea Arcangeli Cc: Larry Woodman Cc: Lee Schermerhorn Acked-by: Fengguang Wu Acked-by: Mel Gorman Signed-off-by: Andi Kleen --- include/linux/rmap.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/linux/rmap.h') diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 77216742c17..9d50e7ef5f5 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -140,6 +140,11 @@ void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned lon void page_add_file_rmap(struct page *); void page_remove_rmap(struct page *); +void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *, + unsigned long); +void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *, + unsigned long); + static inline void page_dup_rmap(struct page *page) { atomic_inc(&page->_mapcount); -- cgit v1.2.3