arm64: cmpxchg: implement cmpxchg64_relaxed

[~andy/linux] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 42c59300bacdff2b75862a7df2bb31a7b9d32c63..dd886fac451ab6ab7d6a3132fd2587c10538f300 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -56,6 +56,7 @@
 #include <linux/ftrace_event.h>
 #include <linux/memcontrol.h>
 #include <linux/prefetch.h>
+#include <linux/mm_inline.h>
 #include <linux/migrate.h>
 #include <linux/page-debug-flags.h>
 #include <linux/hugetlb.h>
@@ -488,8 +489,10 @@ __find_buddy_index(unsigned long page_idx, unsigned int order)
  * (c) a page and its buddy have the same order &&
  * (d) a page and its buddy are in the same zone.
  *
- * For recording whether a page is in the buddy system, we set ->_mapcount -2.
- * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
+ * For recording whether a page is in the buddy system, we set ->_mapcount
+ * PAGE_BUDDY_MAPCOUNT_VALUE.
+ * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
+ * serialized by zone->lock.
  *
  * For recording page's order, we use page_private(page).
  */
@@ -527,8 +530,9 @@ static inline int page_is_buddy(struct page *page, struct page *buddy,
  * as necessary, plus some accounting needed to play nicely with other
  * parts of the VM system.
  * At each level, we keep a list of pages, which are heads of continuous
- * free pages of length of (1 << order) and marked with _mapcount -2. Page's
- * order is recorded in page_private(page) field.
+ * free pages of length of (1 << order) and marked with _mapcount
+ * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
+ * field.
  * So when we are allocating or freeing one, we can derive the state of the
  * other.  That is, if we allocate a small block, and both were
  * free, the remainder of the region must be split into blocks.
@@ -647,7 +651,6 @@ static void free_pcppages_bulk(struct zone *zone, int count,
        int to_free = count;
 
        spin_lock(&zone->lock);
-       zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
 
        while (to_free) {
@@ -696,7 +699,6 @@ static void free_one_page(struct zone *zone, struct page *page, int order,
                                int migratetype)
 {
        spin_lock(&zone->lock);
-       zone->all_unreclaimable = 0;
        zone->pages_scanned = 0;
 
        __free_one_page(page, zone, order, migratetype);
@@ -1306,7 +1308,7 @@ void mark_free_pages(struct zone *zone)
        int order, t;
        struct list_head *curr;
 
-       if (!zone->spanned_pages)
+       if (zone_is_empty(zone))
                return;
 
        spin_lock_irqsave(&zone->lock, flags);
@@ -1896,7 +1898,7 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
 zonelist_scan:
        /*
         * Scan zonelist, looking for a zone with enough free.
-        * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
+        * See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c.
         */
        for_each_zone_zonelist_nodemask(zone, z, zonelist,
                                                high_zoneidx, nodemask) {
@@ -3164,7 +3166,7 @@ void show_free_areas(unsigned int filter)
                        K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
                        K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
                        zone->pages_scanned,
-                       (zone->all_unreclaimable ? "yes" : "no")
+                       (!zone_reclaimable(zone) ? "yes" : "no")
                        );
                printk("lowmem_reserve[]:");
                for (i = 0; i < MAX_NR_ZONES; i++)
@@ -4306,7 +4308,7 @@ int __meminit init_currently_empty_zone(struct zone *zone,
 int __meminit __early_pfn_to_nid(unsigned long pfn)
 {
        unsigned long start_pfn, end_pfn;
-       int i, nid;
+       int nid;
        /*
         * NOTE: The following SMP-unsafe globals are only used early in boot
         * when the kernel is running single-threaded.
@@ -4317,15 +4319,14 @@ int __meminit __early_pfn_to_nid(unsigned long pfn)
        if (last_start_pfn <= pfn && pfn < last_end_pfn)
                return last_nid;
 
-       for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
-               if (start_pfn <= pfn && pfn < end_pfn) {
-                       last_start_pfn = start_pfn;
-                       last_end_pfn = end_pfn;
-                       last_nid = nid;
-                       return nid;
-               }
-       /* This is a memory hole */
-       return -1;
+       nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
+       if (nid != -1) {
+               last_start_pfn = start_pfn;
+               last_end_pfn = end_pfn;
+               last_nid = nid;
+       }
+
+       return nid;
 }
 #endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
 
@@ -5435,7 +5436,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self,
                 * This is only okay since the processor is dead and cannot
                 * race with what we are doing.
                 */
-               refresh_cpu_vm_stats(cpu);
+               cpu_vm_stats_fold(cpu);
        }
        return NOTIFY_OK;
 }
@@ -6008,6 +6009,17 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
                        continue;
 
                page = pfn_to_page(check);
+
+               /*
+                * Hugepages are not in LRU lists, but they're movable.
+                * We need not scan over tail pages bacause we don't
+                * handle each tail page individually in migration.
+                */
+               if (PageHuge(page)) {
+                       iter = round_up(iter + 1, 1<<compound_order(page)) - 1;
+                       continue;
+               }
+
                /*
                 * We can't use page_count without pin a page
                 * because another CPU can free compound page.
@@ -6354,10 +6366,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
                list_del(&page->lru);
                rmv_page_order(page);
                zone->free_area[order].nr_free--;
-#ifdef CONFIG_HIGHMEM
-               if (PageHighMem(page))
-                       totalhigh_pages -= 1 << order;
-#endif
                for (i = 0; i < (1 << order); i++)
                        SetPageReserved((page+i));
                pfn += (1 << order);