mm: direct IO starvation improvement

[~andy/linux] / mm / swapfile.c

diff --git a/mm/swapfile.c b/mm/swapfile.c
index ca75b9e7c09f6efd864fe78f45ec95627bef903d..f28745855772506fb34ed19252e130343daeb532 100644 (file)
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -16,6 +16,7 @@
 #include <linux/namei.h>
 #include <linux/shm.h>
 #include <linux/blkdev.h>
+#include <linux/random.h>
 #include <linux/writeback.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
@@ -95,7 +96,7 @@ static int discard_swap(struct swap_info_struct *si)
 
        list_for_each_entry(se, &si->extent_list, list) {
                sector_t start_block = se->start_block << (PAGE_SHIFT - 9);
-               pgoff_t nr_blocks = se->nr_pages << (PAGE_SHIFT - 9);
+               sector_t nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
 
                if (se->start_page == 0) {
                        /* Do not discard the swap header page! */
@@ -132,7 +133,7 @@ static void discard_swap_cluster(struct swap_info_struct *si,
                    start_page < se->start_page + se->nr_pages) {
                        pgoff_t offset = start_page - se->start_page;
                        sector_t start_block = se->start_block + offset;
-                       pgoff_t nr_blocks = se->nr_pages - offset;
+                       sector_t nr_blocks = se->nr_pages - offset;
 
                        if (nr_blocks > nr_pages)
                                nr_blocks = nr_pages;
@@ -168,6 +169,7 @@ static int wait_for_discard(void *word)
 static inline unsigned long scan_swap_map(struct swap_info_struct *si)
 {
        unsigned long offset;
+       unsigned long scan_base;
        unsigned long last_in_cluster = 0;
        int latency_ration = LATENCY_LIMIT;
        int found_free_cluster = 0;
@@ -180,10 +182,11 @@ static inline unsigned long scan_swap_map(struct swap_info_struct *si)
         * all over the entire swap partition, so that we reduce
         * overall disk seek times between swap pages.  -- sct
         * But we do now try to find an empty cluster.  -Andrea
+        * And we let swap pages go all over an SSD partition.  Hugh
         */
 
        si->flags += SWP_SCANNING;
-       offset = si->cluster_next;
+       scan_base = offset = si->cluster_next;
 
        if (unlikely(!si->cluster_nr--)) {
                if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
@@ -205,7 +208,16 @@ static inline unsigned long scan_swap_map(struct swap_info_struct *si)
                }
                spin_unlock(&swap_lock);
 
-               offset = si->lowest_bit;
+               /*
+                * If seek is expensive, start searching for new cluster from
+                * start of partition, to minimize the span of allocated swap.
+                * But if seek is cheap, search from our current position, so
+                * that swap is allocated from all over the partition: if the
+                * Flash Translation Layer only remaps within limited zones,
+                * we don't want to wear out the first zone too quickly.
+                */
+               if (!(si->flags & SWP_SOLIDSTATE))
+                       scan_base = offset = si->lowest_bit;
                last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
 
                /* Locate the first empty (unaligned) cluster */
@@ -227,6 +239,27 @@ static inline unsigned long scan_swap_map(struct swap_info_struct *si)
                }
 
                offset = si->lowest_bit;
+               last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
+
+               /* Locate the first empty (unaligned) cluster */
+               for (; last_in_cluster < scan_base; offset++) {
+                       if (si->swap_map[offset])
+                               last_in_cluster = offset + SWAPFILE_CLUSTER;
+                       else if (offset == last_in_cluster) {
+                               spin_lock(&swap_lock);
+                               offset -= SWAPFILE_CLUSTER - 1;
+                               si->cluster_next = offset;
+                               si->cluster_nr = SWAPFILE_CLUSTER - 1;
+                               found_free_cluster = 1;
+                               goto checks;
+                       }
+                       if (unlikely(--latency_ration < 0)) {
+                               cond_resched();
+                               latency_ration = LATENCY_LIMIT;
+                       }
+               }
+
+               offset = scan_base;
                spin_lock(&swap_lock);
                si->cluster_nr = SWAPFILE_CLUSTER - 1;
                si->lowest_alloc = 0;
@@ -238,7 +271,7 @@ checks:
        if (!si->highest_bit)
                goto no_page;
        if (offset > si->highest_bit)
-               offset = si->lowest_bit;
+               scan_base = offset = si->lowest_bit;
        if (si->swap_map[offset])
                goto scan;
 
@@ -322,8 +355,18 @@ scan:
                        latency_ration = LATENCY_LIMIT;
                }
        }
+       offset = si->lowest_bit;
+       while (++offset < scan_base) {
+               if (!si->swap_map[offset]) {
+                       spin_lock(&swap_lock);
+                       goto checks;
+               }
+               if (unlikely(--latency_ration < 0)) {
+                       cond_resched();
+                       latency_ration = LATENCY_LIMIT;
+               }
+       }
        spin_lock(&swap_lock);
-       goto checks;
 
 no_page:
        si->flags -= SWP_SCANNING;
@@ -528,13 +571,13 @@ int try_to_free_swap(struct page *page)
  * Free the swap entry like above, but also try to
  * free the page cache entry if it is the last user.
  */
-void free_swap_and_cache(swp_entry_t entry)
+int free_swap_and_cache(swp_entry_t entry)
 {
-       struct swap_info_struct * p;
+       struct swap_info_struct *p;
        struct page *page = NULL;
 
        if (is_migration_entry(entry))
-               return;
+               return 1;
 
        p = swap_info_get(entry);
        if (p) {
@@ -560,6 +603,7 @@ void free_swap_and_cache(swp_entry_t entry)
                unlock_page(page);
                page_cache_release(page);
        }
+       return p != NULL;
 }
 
 #ifdef CONFIG_HIBERNATION
@@ -1806,6 +1850,10 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
                goto bad_swap;
        }
 
+       if (blk_queue_nonrot(bdev_get_queue(p->bdev))) {
+               p->flags |= SWP_SOLIDSTATE;
+               p->cluster_next = 1 + (random32() % p->highest_bit);
+       }
        if (discard_swap(p) == 0)
                p->flags |= SWP_DISCARDABLE;
 
@@ -1822,10 +1870,11 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
        total_swap_pages += nr_good_pages;
 
        printk(KERN_INFO "Adding %uk swap on %s.  "
-                       "Priority:%d extents:%d across:%lluk%s\n",
+                       "Priority:%d extents:%d across:%lluk %s%s\n",
                nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
                nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
-               (p->flags & SWP_DISCARDABLE) ? " D" : "");
+               (p->flags & SWP_SOLIDSTATE) ? "SS" : "",
+               (p->flags & SWP_DISCARDABLE) ? "D" : "");
 
        /* insert swap space into swap_list: */
        prev = -1;