mm: page_alloc: fair zone allocator policy

[~andy/linux] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9884aa0f233a349c4ddc06cb10a45bbbed44b7a3..544d19d681a2bedb8f3476b26e08bce28a6c0ce8 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1551,6 +1551,7 @@ again:
                                          get_pageblock_migratetype(page));
        }
 
+       __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order));
        __count_zone_vm_events(PGALLOC, zone, 1 << order);
        zone_statistics(preferred_zone, zone, gfp_flags);
        local_irq_restore(flags);
@@ -1817,6 +1818,11 @@ static void zlc_clear_zones_full(struct zonelist *zonelist)
        bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
 }
 
+static bool zone_local(struct zone *local_zone, struct zone *zone)
+{
+       return node_distance(local_zone->node, zone->node) == LOCAL_DISTANCE;
+}
+
 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
 {
        return node_isset(local_zone->node, zone->zone_pgdat->reclaim_nodes);
@@ -1854,6 +1860,11 @@ static void zlc_clear_zones_full(struct zonelist *zonelist)
 {
 }
 
+static bool zone_local(struct zone *local_zone, struct zone *zone)
+{
+       return true;
+}
+
 static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
 {
        return true;
@@ -1900,6 +1911,26 @@ zonelist_scan:
                BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
                if (alloc_flags & ALLOC_NO_WATERMARKS)
                        goto try_this_zone;
+               /*
+                * Distribute pages in proportion to the individual
+                * zone size to ensure fair page aging.  The zone a
+                * page was allocated in should have no effect on the
+                * time the page has in memory before being reclaimed.
+                *
+                * When zone_reclaim_mode is enabled, try to stay in
+                * local zones in the fastpath.  If that fails, the
+                * slowpath is entered, which will do another pass
+                * starting with the local zones, but ultimately fall
+                * back to remote zones that do not partake in the
+                * fairness round-robin cycle of this zonelist.
+                */
+               if (alloc_flags & ALLOC_WMARK_LOW) {
+                       if (zone_page_state(zone, NR_ALLOC_BATCH) <= 0)
+                               continue;
+                       if (zone_reclaim_mode &&
+                           !zone_local(preferred_zone, zone))
+                               continue;
+               }
                /*
                 * When allocating a page cache page for writing, we
                 * want to get it from a zone that is within its dirty
@@ -2346,16 +2377,30 @@ __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
        return page;
 }
 
-static inline
-void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
-                                               enum zone_type high_zoneidx,
-                                               enum zone_type classzone_idx)
+static void prepare_slowpath(gfp_t gfp_mask, unsigned int order,
+                            struct zonelist *zonelist,
+                            enum zone_type high_zoneidx,
+                            struct zone *preferred_zone)
 {
        struct zoneref *z;
        struct zone *zone;
 
-       for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
-               wakeup_kswapd(zone, order, classzone_idx);
+       for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
+               if (!(gfp_mask & __GFP_NO_KSWAPD))
+                       wakeup_kswapd(zone, order, zone_idx(preferred_zone));
+               /*
+                * Only reset the batches of zones that were actually
+                * considered in the fast path, we don't want to
+                * thrash fairness information for zones that are not
+                * actually part of this zonelist's round-robin cycle.
+                */
+               if (zone_reclaim_mode && !zone_local(preferred_zone, zone))
+                       continue;
+               mod_zone_page_state(zone, NR_ALLOC_BATCH,
+                                   high_wmark_pages(zone) -
+                                   low_wmark_pages(zone) -
+                                   zone_page_state(zone, NR_ALLOC_BATCH));
+       }
 }
 
 static inline int
@@ -2451,9 +2496,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
                goto nopage;
 
 restart:
-       if (!(gfp_mask & __GFP_NO_KSWAPD))
-               wake_all_kswapd(order, zonelist, high_zoneidx,
-                                               zone_idx(preferred_zone));
+       prepare_slowpath(gfp_mask, order, zonelist,
+                        high_zoneidx, preferred_zone);
 
        /*
         * OK, we're below the kswapd watermark and have kicked background
@@ -4753,8 +4797,11 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                spin_lock_init(&zone->lru_lock);
                zone_seqlock_init(zone);
                zone->zone_pgdat = pgdat;
-
                zone_pcp_init(zone);
+
+               /* For bootup, initialized properly in watermark setup */
+               mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages);
+
                lruvec_init(&zone->lruvec);
                if (!size)
                        continue;
@@ -5525,6 +5572,11 @@ static void __setup_per_zone_wmarks(void)
                zone->watermark[WMARK_LOW]  = min_wmark_pages(zone) + (tmp >> 2);
                zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
 
+               __mod_zone_page_state(zone, NR_ALLOC_BATCH,
+                                     high_wmark_pages(zone) -
+                                     low_wmark_pages(zone) -
+                                     zone_page_state(zone, NR_ALLOC_BATCH));
+
                setup_zone_migrate_reserve(zone);
                spin_unlock_irqrestore(&zone->lock, flags);
        }