slab: propagate tunable values

SLAB allows us to tune a particular cache behavior with tunables.  When
creating a new memcg cache copy, we'd like to preserve any tunables the
parent cache already had.

This could be done by an explicit call to do_tune_cpucache() after the
cache is created.  But this is not very convenient now that the caches are
created from common code, since this function is SLAB-specific.

Another method of doing that is taking advantage of the fact that
do_tune_cpucache() is always called from enable_cpucache(), which is
called at cache initialization.  We can just preset the values, and then
things work as expected.

It can also happen that a root cache has its tunables updated during
normal system operation.  In this case, we will propagate the change to
all caches that are already active.

This change will require us to move the assignment of root_cache in
memcg_params a bit earlier.  We need this to be already set - which
memcg_kmem_register_cache will do - when we reach __kmem_cache_create()

Signed-off-by: Glauber Costa <glommer@parallels.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Frederic Weisbecker <fweisbec@redhat.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: JoonSoo Kim <js1304@gmail.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Michal Hocko <mhocko@suse.cz>
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Cc: Rik van Riel <riel@redhat.com>
Cc: Suleiman Souhlal <suleiman@google.com>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 8dc7c746b44fb77173b2f27425e675bb09991dcf..ea02ff9708363a0e9893af1b8d668cc7cee86ec0 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -448,7 +448,8 @@ void __memcg_kmem_commit_charge(struct page *page,
 void __memcg_kmem_uncharge_pages(struct page *page, int order);
 
 int memcg_cache_id(struct mem_cgroup *memcg);
 void __memcg_kmem_uncharge_pages(struct page *page, int order);
 
 int memcg_cache_id(struct mem_cgroup *memcg);

-int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s);
+int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
+                        struct kmem_cache *root_cache);

 void memcg_release_cache(struct kmem_cache *cachep);
 void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep);
 
 void memcg_release_cache(struct kmem_cache *cachep);
 void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep);
 


@@ -590,8 +591,9 @@ static inline int memcg_cache_id(struct mem_cgroup *memcg)
        return -1;
 }
 
        return -1;
 }
 

-static inline int memcg_register_cache(struct mem_cgroup *memcg,
-                                      struct kmem_cache *s)
+static inline int
+memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
+                    struct kmem_cache *root_cache)

 {
        return 0;
 }
 {
        return 0;
 }

diff --git a/include/linux/slab.h b/include/linux/slab.h
index b9278663f22a0842219b82d98d7db047392f1a59..5d168d7e0a28f4bbd37331de4b732db5cecdffaf 100644 (file)
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -130,7 +130,7 @@ struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
                        void (*)(void *));
 struct kmem_cache *
 kmem_cache_create_memcg(struct mem_cgroup *, const char *, size_t, size_t,
                        void (*)(void *));
 struct kmem_cache *
 kmem_cache_create_memcg(struct mem_cgroup *, const char *, size_t, size_t,

-                       unsigned long, void (*)(void *));
+                       unsigned long, void (*)(void *), struct kmem_cache *);

 void kmem_cache_destroy(struct kmem_cache *);
 int kmem_cache_shrink(struct kmem_cache *);
 void kmem_cache_free(struct kmem_cache *, void *);
 void kmem_cache_destroy(struct kmem_cache *);
 int kmem_cache_shrink(struct kmem_cache *);
 void kmem_cache_free(struct kmem_cache *, void *);

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a32d83c2e353f64db62519baeb738d8ddb90280a..f3009b4bae510bdf5eced41a429aa1476519df2f 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3012,7 +3012,8 @@ int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
        return 0;
 }
 
        return 0;
 }
 

-int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s)
+int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
+                        struct kmem_cache *root_cache)

 {
        size_t size = sizeof(struct memcg_cache_params);
 
 {
        size_t size = sizeof(struct memcg_cache_params);
 


@@ -3026,8 +3027,10 @@ int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s)
        if (!s->memcg_params)
                return -ENOMEM;
 
        if (!s->memcg_params)
                return -ENOMEM;
 

-       if (memcg)
+       if (memcg) {

                s->memcg_params->memcg = memcg;
                s->memcg_params->memcg = memcg;

+               s->memcg_params->root_cache = root_cache;
+       }

        return 0;
 }
 
        return 0;
 }
 


@@ -3186,7 +3189,7 @@ static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
                return NULL;
 
        new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
                return NULL;
 
        new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,

-                                     (s->flags & ~SLAB_PANIC), s->ctor);
+                                     (s->flags & ~SLAB_PANIC), s->ctor, s);

 
        if (new)
                new->allocflags |= __GFP_KMEMCG;
 
        if (new)
                new->allocflags |= __GFP_KMEMCG;


@@ -3226,7 +3229,6 @@ static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
        }
 
        mem_cgroup_get(memcg);
        }
 
        mem_cgroup_get(memcg);

-       new_cachep->memcg_params->root_cache = cachep;

        atomic_set(&new_cachep->memcg_params->nr_pages , 0);
 
        cachep->memcg_params->memcg_caches[idx] = new_cachep;
        atomic_set(&new_cachep->memcg_params->nr_pages , 0);
 
        cachep->memcg_params->memcg_caches[idx] = new_cachep;

diff --git a/mm/slab.c b/mm/slab.c
index 7467343f9fe7653456c4e6f469511ec63b3dfe0e..4dcbf96a77b47987f5a7ac4b94e5767ca1b288db 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -4041,7 +4041,7 @@ static void do_ccupdate_local(void *info)
 }
 
 /* Always called with the slab_mutex held */
 }
 
 /* Always called with the slab_mutex held */

-static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
+static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,

                                int batchcount, int shared, gfp_t gfp)
 {
        struct ccupdate_struct *new;
                                int batchcount, int shared, gfp_t gfp)
 {
        struct ccupdate_struct *new;


@@ -4084,12 +4084,48 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
        return alloc_kmemlist(cachep, gfp);
 }
 
        return alloc_kmemlist(cachep, gfp);
 }
 

+static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
+                               int batchcount, int shared, gfp_t gfp)
+{
+       int ret;
+       struct kmem_cache *c = NULL;
+       int i = 0;
+
+       ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
+
+       if (slab_state < FULL)
+               return ret;
+
+       if ((ret < 0) || !is_root_cache(cachep))
+               return ret;
+
+       for_each_memcg_cache_index(i) {
+               c = cache_from_memcg(cachep, i);
+               if (c)
+                       /* return value determined by the parent cache only */
+                       __do_tune_cpucache(c, limit, batchcount, shared, gfp);
+       }
+
+       return ret;
+}
+

 /* Called with slab_mutex held always */
 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
 {
        int err;
 /* Called with slab_mutex held always */
 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
 {
        int err;

-       int limit, shared;
+       int limit = 0;
+       int shared = 0;
+       int batchcount = 0;
+
+       if (!is_root_cache(cachep)) {
+               struct kmem_cache *root = memcg_root_cache(cachep);
+               limit = root->limit;
+               shared = root->shared;
+               batchcount = root->batchcount;
+       }

 
 

+       if (limit && shared && batchcount)
+               goto skip_setup;

        /*
         * The head array serves three purposes:
         * - create a LIFO ordering, i.e. return objects that are cache-warm
        /*
         * The head array serves three purposes:
         * - create a LIFO ordering, i.e. return objects that are cache-warm


@@ -4131,7 +4167,9 @@ static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
        if (limit > 32)
                limit = 32;
 #endif
        if (limit > 32)
                limit = 32;
 #endif

-       err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared, gfp);
+       batchcount = (limit + 1) / 2;
+skip_setup:
+       err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);

        if (err)
                printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
                       cachep->name, -err);
        if (err)
                printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
                       cachep->name, -err);

diff --git a/mm/slab.h b/mm/slab.h
index ec5dae1c8e75100654b93eb03f2d627ed6a26194..34a98d642196f5e57e2bc2a1d94f7333b3ab7c48 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -155,6 +155,13 @@ static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
 {
        return s->memcg_params->memcg_caches[idx];
 }
 {
        return s->memcg_params->memcg_caches[idx];
 }

+
+static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
+{
+       if (is_root_cache(s))
+               return s;
+       return s->memcg_params->root_cache;
+}

 #else
 static inline bool is_root_cache(struct kmem_cache *s)
 {
 #else
 static inline bool is_root_cache(struct kmem_cache *s)
 {


@@ -190,6 +197,11 @@ static inline struct kmem_cache *cache_from_memcg(struct kmem_cache *s, int idx)
 {
        return NULL;
 }
 {
        return NULL;
 }

+
+static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
+{
+       return s;
+}

 #endif
 
 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
 #endif
 
 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)

diff --git a/mm/slab_common.c b/mm/slab_common.c
index 081f1b8d9a7b21ae5fb4824fd3e08200fc7f0071..3f3cd97d3fdf6503ef7a1dda55ed16fb9d6f57e4 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -164,7 +164,8 @@ unsigned long calculate_alignment(unsigned long flags,
 
 struct kmem_cache *
 kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
 
 struct kmem_cache *
 kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,

-                       size_t align, unsigned long flags, void (*ctor)(void *))
+                       size_t align, unsigned long flags, void (*ctor)(void *),
+                       struct kmem_cache *parent_cache)

 {
        struct kmem_cache *s = NULL;
        int err = 0;
 {
        struct kmem_cache *s = NULL;
        int err = 0;


@@ -193,7 +194,7 @@ kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
                s->align = calculate_alignment(flags, align, size);
                s->ctor = ctor;
 
                s->align = calculate_alignment(flags, align, size);
                s->ctor = ctor;
 

-               if (memcg_register_cache(memcg, s)) {
+               if (memcg_register_cache(memcg, s, parent_cache)) {

                        kmem_cache_free(kmem_cache, s);
                        err = -ENOMEM;
                        goto out_locked;
                        kmem_cache_free(kmem_cache, s);
                        err = -ENOMEM;
                        goto out_locked;


@@ -243,7 +244,7 @@ struct kmem_cache *
 kmem_cache_create(const char *name, size_t size, size_t align,
                  unsigned long flags, void (*ctor)(void *))
 {
 kmem_cache_create(const char *name, size_t size, size_t align,
                  unsigned long flags, void (*ctor)(void *))
 {

-       return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor);
+       return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor, NULL);

 }
 EXPORT_SYMBOL(kmem_cache_create);
 
 }
 EXPORT_SYMBOL(kmem_cache_create);