Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jmorris...

[~andy/linux] / mm / huge_memory.c

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index f60c4ebaa30c6147dafb4ba650f388c62a47edc4..610e3df2768a6a5b2ec1e293da4c96dafbbe2d30 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -211,24 +211,29 @@ static void put_huge_zero_page(void)
        BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
 }
 
-static int shrink_huge_zero_page(struct shrinker *shrink,
-               struct shrink_control *sc)
+static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
+                                       struct shrink_control *sc)
 {
-       if (!sc->nr_to_scan)
-               /* we can free zero page only if last reference remains */
-               return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
+       /* we can free zero page only if last reference remains */
+       return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
+}
 
+static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
+                                      struct shrink_control *sc)
+{
        if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
                struct page *zero_page = xchg(&huge_zero_page, NULL);
                BUG_ON(zero_page == NULL);
                __free_page(zero_page);
+               return HPAGE_PMD_NR;
        }
 
        return 0;
 }
 
 static struct shrinker huge_zero_page_shrinker = {
-       .shrink = shrink_huge_zero_page,
+       .count_objects = shrink_huge_zero_page_count,
+       .scan_objects = shrink_huge_zero_page_scan,
        .seeks = DEFAULT_SEEKS,
 };
 
@@ -2692,6 +2697,7 @@ void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
 
        mmun_start = haddr;
        mmun_end   = haddr + HPAGE_PMD_SIZE;
+again:
        mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
        spin_lock(&mm->page_table_lock);
        if (unlikely(!pmd_trans_huge(*pmd))) {
@@ -2714,7 +2720,14 @@ void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address,
        split_huge_page(page);
 
        put_page(page);
-       BUG_ON(pmd_trans_huge(*pmd));
+
+       /*
+        * We don't always have down_write of mmap_sem here: a racing
+        * do_huge_pmd_wp_page() might have copied-on-write to another
+        * huge page before our split_huge_page() got the anon_vma lock.
+        */
+       if (unlikely(pmd_trans_huge(*pmd)))
+               goto again;
 }
 
 void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,