Merge branch 'slab/next' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg...

[~andy/linux] / include / linux / mm_types.h

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 95bf0c5a7eb9584cd668b9fb60268543dcf91dbb..bd299418a934e21b99c303af82a7c2f427bbf915 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -23,7 +23,9 @@
 
 struct address_space;
 
-#define USE_SPLIT_PTLOCKS      (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
+#define USE_SPLIT_PTE_PTLOCKS  (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
+#define USE_SPLIT_PMD_PTLOCKS  (USE_SPLIT_PTE_PTLOCKS && \
+               IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
 
 /*
  * Each physical page in the system has a struct page associated with
@@ -138,6 +140,9 @@ struct page {
                struct rcu_head rcu_head;       /* Used by SLAB
                                                 * when destroying via RCU
                                                 */
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
+               pgtable_t pmd_huge_pte; /* protected by page->ptl */
+#endif
        };
 
        /* Remainder is not double word aligned */
@@ -149,8 +154,12 @@ struct page {
                                                 * indicates order in the buddy
                                                 * system if PG_buddy is set.
                                                 */
-#if USE_SPLIT_PTLOCKS
+#if USE_SPLIT_PTE_PTLOCKS
+#if BLOATED_SPINLOCKS
+               spinlock_t *ptl;
+#else
                spinlock_t ptl;
+#endif
 #endif
                struct kmem_cache *slab_cache;  /* SL[AU]B: Pointer to slab */
                struct page *first_page;        /* Compound tail pages */
@@ -182,8 +191,8 @@ struct page {
        void *shadow;
 #endif
 
-#ifdef LAST_NID_NOT_IN_PAGE_FLAGS
-       int _last_nid;
+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
+       int _last_cpupid;
 #endif
 }
 /*
@@ -317,19 +326,20 @@ enum {
        NR_MM_COUNTERS
 };
 
-#if USE_SPLIT_PTLOCKS && defined(CONFIG_MMU)
+#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
 #define SPLIT_RSS_COUNTING
 /* per-thread cached information, */
 struct task_rss_stat {
        int events;     /* for synchronization threshold */
        int count[NR_MM_COUNTERS];
 };
-#endif /* USE_SPLIT_PTLOCKS */
+#endif /* USE_SPLIT_PTE_PTLOCKS */
 
 struct mm_rss_stat {
        atomic_long_t count[NR_MM_COUNTERS];
 };
 
+struct kioctx_table;
 struct mm_struct {
        struct vm_area_struct * mmap;           /* list of VMAs */
        struct rb_root mm_rb;
@@ -346,6 +356,7 @@ struct mm_struct {
        pgd_t * pgd;
        atomic_t mm_users;                      /* How many users with user space? */
        atomic_t mm_count;                      /* How many references to "struct mm_struct" (users count as 1) */
+       atomic_long_t nr_ptes;                  /* Page table pages */
        int map_count;                          /* number of VMAs */
 
        spinlock_t page_table_lock;             /* Protects page tables and some counters */
@@ -367,7 +378,6 @@ struct mm_struct {
        unsigned long exec_vm;          /* VM_EXEC & ~VM_WRITE */
        unsigned long stack_vm;         /* VM_GROWSUP/DOWN */
        unsigned long def_flags;
-       unsigned long nr_ptes;          /* Page table pages */
        unsigned long start_code, end_code, start_data, end_data;
        unsigned long start_brk, brk, start_stack;
        unsigned long arg_start, arg_end, env_start, env_end;
@@ -391,8 +401,8 @@ struct mm_struct {
 
        struct core_state *core_state; /* coredumping support */
 #ifdef CONFIG_AIO
-       spinlock_t              ioctx_lock;
-       struct hlist_head       ioctx_list;
+       spinlock_t                      ioctx_lock;
+       struct kioctx_table __rcu       *ioctx_table;
 #endif
 #ifdef CONFIG_MM_OWNER
        /*
@@ -413,7 +423,7 @@ struct mm_struct {
 #ifdef CONFIG_MMU_NOTIFIER
        struct mmu_notifier_mm *mmu_notifier_mm;
 #endif
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
        pgtable_t pmd_huge_pte; /* protected by page_table_lock */
 #endif
 #ifdef CONFIG_CPUMASK_OFFSTACK
@@ -427,28 +437,15 @@ struct mm_struct {
         */
        unsigned long numa_next_scan;
 
-       /* numa_next_reset is when the PTE scanner period will be reset */
-       unsigned long numa_next_reset;
-
        /* Restart point for scanning and setting pte_numa */
        unsigned long numa_scan_offset;
 
        /* numa_scan_seq prevents two threads setting pte_numa */
        int numa_scan_seq;
-
-       /*
-        * The first node a task was scheduled on. If a task runs on
-        * a different node than Make PTE Scan Go Now.
-        */
-       int first_nid;
 #endif
        struct uprobes_state uprobes_state;
 };
 
-/* first nid will either be a valid NID or one of these values */
-#define NUMA_PTE_SCAN_INIT     -1
-#define NUMA_PTE_SCAN_ACTIVE   -2
-
 static inline void mm_init_cpumask(struct mm_struct *mm)
 {
 #ifdef CONFIG_CPUMASK_OFFSTACK