Merge tag 'upstream-3.4-rc1' of git://git.infradead.org/linux-ubifs

[~andy/linux] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c200875072f7ffe23f66fc3e61dd31d70ce2b516..b2ee6df0e9bb31eebd3b2f1528cbbcccb2b9c43e 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3902,7 +3902,6 @@ static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
                break;
        default:
                BUG();
-               break;
        }
        return val;
 }
@@ -4467,12 +4466,6 @@ static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
        else
                BUG();
 
-       /*
-        * Something went wrong if we trying to unregister a threshold
-        * if we don't have thresholds
-        */
-       BUG_ON(!thresholds);
-
        if (!thresholds->primary)
                goto unlock;
 
@@ -5117,7 +5110,7 @@ one_by_one:
 }
 
 /**
- * is_target_pte_for_mc - check a pte whether it is valid for move charge
+ * get_mctgt_type - get target type of moving charge
  * @vma: the vma the pte to be checked belongs
  * @addr: the address corresponding to the pte to be checked
  * @ptent: the pte to be checked
@@ -5140,7 +5133,7 @@ union mc_target {
 };
 
 enum mc_target_type {
-       MC_TARGET_NONE, /* not used */
+       MC_TARGET_NONE = 0,
        MC_TARGET_PAGE,
        MC_TARGET_SWAP,
 };
@@ -5221,12 +5214,12 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
        return page;
 }
 
-static int is_target_pte_for_mc(struct vm_area_struct *vma,
+static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
                unsigned long addr, pte_t ptent, union mc_target *target)
 {
        struct page *page = NULL;
        struct page_cgroup *pc;
-       int ret = 0;
+       enum mc_target_type ret = MC_TARGET_NONE;
        swp_entry_t ent = { .val = 0 };
 
        if (pte_present(ptent))
@@ -5237,7 +5230,7 @@ static int is_target_pte_for_mc(struct vm_area_struct *vma,
                page = mc_handle_file_pte(vma, addr, ptent, &ent);
 
        if (!page && !ent.val)
-               return 0;
+               return ret;
        if (page) {
                pc = lookup_page_cgroup(page);
                /*
@@ -5263,6 +5256,41 @@ static int is_target_pte_for_mc(struct vm_area_struct *vma,
        return ret;
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+/*
+ * We don't consider swapping or file mapped pages because THP does not
+ * support them for now.
+ * Caller should make sure that pmd_trans_huge(pmd) is true.
+ */
+static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
+               unsigned long addr, pmd_t pmd, union mc_target *target)
+{
+       struct page *page = NULL;
+       struct page_cgroup *pc;
+       enum mc_target_type ret = MC_TARGET_NONE;
+
+       page = pmd_page(pmd);
+       VM_BUG_ON(!page || !PageHead(page));
+       if (!move_anon())
+               return ret;
+       pc = lookup_page_cgroup(page);
+       if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
+               ret = MC_TARGET_PAGE;
+               if (target) {
+                       get_page(page);
+                       target->page = page;
+               }
+       }
+       return ret;
+}
+#else
+static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
+               unsigned long addr, pmd_t pmd, union mc_target *target)
+{
+       return MC_TARGET_NONE;
+}
+#endif
+
 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
                                        unsigned long addr, unsigned long end,
                                        struct mm_walk *walk)
@@ -5271,13 +5299,16 @@ static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
        pte_t *pte;
        spinlock_t *ptl;
 
-       split_huge_page_pmd(walk->mm, pmd);
-       if (pmd_trans_unstable(pmd))
+       if (pmd_trans_huge_lock(pmd, vma) == 1) {
+               if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
+                       mc.precharge += HPAGE_PMD_NR;
+               spin_unlock(&vma->vm_mm->page_table_lock);
                return 0;
+       }
 
        pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
        for (; addr != end; pte++, addr += PAGE_SIZE)
-               if (is_target_pte_for_mc(vma, addr, *pte, NULL))
+               if (get_mctgt_type(vma, addr, *pte, NULL))
                        mc.precharge++; /* increment precharge temporarily */
        pte_unmap_unlock(pte - 1, ptl);
        cond_resched();
@@ -5432,25 +5463,55 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
        struct vm_area_struct *vma = walk->private;
        pte_t *pte;
        spinlock_t *ptl;
+       enum mc_target_type target_type;
+       union mc_target target;
+       struct page *page;
+       struct page_cgroup *pc;
 
-       split_huge_page_pmd(walk->mm, pmd);
-       if (pmd_trans_unstable(pmd))
+       /*
+        * We don't take compound_lock() here but no race with splitting thp
+        * happens because:
+        *  - if pmd_trans_huge_lock() returns 1, the relevant thp is not
+        *    under splitting, which means there's no concurrent thp split,
+        *  - if another thread runs into split_huge_page() just after we
+        *    entered this if-block, the thread must wait for page table lock
+        *    to be unlocked in __split_huge_page_splitting(), where the main
+        *    part of thp split is not executed yet.
+        */
+       if (pmd_trans_huge_lock(pmd, vma) == 1) {
+               if (!mc.precharge) {
+                       spin_unlock(&vma->vm_mm->page_table_lock);
+                       return 0;
+               }
+               target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
+               if (target_type == MC_TARGET_PAGE) {
+                       page = target.page;
+                       if (!isolate_lru_page(page)) {
+                               pc = lookup_page_cgroup(page);
+                               if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
+                                                            pc, mc.from, mc.to,
+                                                            false)) {
+                                       mc.precharge -= HPAGE_PMD_NR;
+                                       mc.moved_charge += HPAGE_PMD_NR;
+                               }
+                               putback_lru_page(page);
+                       }
+                       put_page(page);
+               }
+               spin_unlock(&vma->vm_mm->page_table_lock);
                return 0;
+       }
+
 retry:
        pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
        for (; addr != end; addr += PAGE_SIZE) {
                pte_t ptent = *(pte++);
-               union mc_target target;
-               int type;
-               struct page *page;
-               struct page_cgroup *pc;
                swp_entry_t ent;
 
                if (!mc.precharge)
                        break;
 
-               type = is_target_pte_for_mc(vma, addr, ptent, &target);
-               switch (type) {
+               switch (get_mctgt_type(vma, addr, ptent, &target)) {
                case MC_TARGET_PAGE:
                        page = target.page;
                        if (isolate_lru_page(page))
@@ -5463,7 +5524,7 @@ retry:
                                mc.moved_charge++;
                        }
                        putback_lru_page(page);
-put:                   /* is_target_pte_for_mc() gets the page */
+put:                   /* get_mctgt_type() gets the page */
                        put_page(page);
                        break;
                case MC_TARGET_SWAP: