[PATCH] lockdep: core, reduce per-lock class-cache size

lockdep_map is embedded into every lock, which blows up data structure
sizes all around the kernel.  Reduce the class-cache to be for the default
class only - that is used in 99.9% of the cases and even if we dont have a
class cached, the lookup in the class-hash is lockless.

This change reduces the per-lock dep_map overhead by 56 bytes on 64-bit
platforms and by 28 bytes on 32-bit platforms.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

2 files changed, 55 insertions, 34 deletions

diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index 316e0fb8d7b1..c040a8c969aa 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -120,7 +120,7 @@ struct lock_class {
  */
 struct lockdep_map {
 	struct lock_class_key		*key;
-	struct lock_class		*class[MAX_LOCKDEP_SUBCLASSES];
+	struct lock_class		*class_cache;
 	const char			*name;
 };
 
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index c1f34addd003..9bad17884513 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -1104,7 +1104,7 @@ extern void __error_too_big_MAX_LOCKDEP_SUBCLASSES(void);
  * itself, so actual lookup of the hash should be once per lock object.
  */
 static inline struct lock_class *
-register_lock_class(struct lockdep_map *lock, unsigned int subclass)
+look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
 {
 	struct lockdep_subclass_key *key;
 	struct list_head *hash_head;
@@ -1148,7 +1148,26 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass)
 	 */
 	list_for_each_entry(class, hash_head, hash_entry)
 		if (class->key == key)
-			goto out_set;
+			return class;
+
+	return NULL;
+}
+
+/*
+ * Register a lock's class in the hash-table, if the class is not present
+ * yet. Otherwise we look it up. We cache the result in the lock object
+ * itself, so actual lookup of the hash should be once per lock object.
+ */
+static inline struct lock_class *
+register_lock_class(struct lockdep_map *lock, unsigned int subclass)
+{
+	struct lockdep_subclass_key *key;
+	struct list_head *hash_head;
+	struct lock_class *class;
+
+	class = look_up_lock_class(lock, subclass);
+	if (likely(class))
+		return class;
 
 	/*
 	 * Debug-check: all keys must be persistent!
@@ -1163,6 +1182,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass)
 		return NULL;
 	}
 
+	key = lock->key->subkeys + subclass;
+	hash_head = classhashentry(key);
+
 	__raw_spin_lock(&hash_lock);
 	/*
 	 * We have to do the hash-walk again, to avoid races
@@ -1209,8 +1231,8 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass)
 out_unlock_set:
 	__raw_spin_unlock(&hash_lock);
 
-out_set:
-	lock->class[subclass] = class;
+	if (!subclass)
+		lock->class_cache = class;
 
 	DEBUG_LOCKS_WARN_ON(class->subclass != subclass);
 
@@ -1914,7 +1936,7 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name,
 	}
 	lock->name = name;
 	lock->key = key;
-	memset(lock->class, 0, sizeof(lock->class[0])*MAX_LOCKDEP_SUBCLASSES);
+	lock->class_cache = NULL;
 }
 
 EXPORT_SYMBOL_GPL(lockdep_init_map);
@@ -1928,8 +1950,8 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 			  unsigned long ip)
 {
 	struct task_struct *curr = current;
+	struct lock_class *class = NULL;
 	struct held_lock *hlock;
-	struct lock_class *class;
 	unsigned int depth, id;
 	int chain_head = 0;
 	u64 chain_key;
@@ -1947,8 +1969,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
 		return 0;
 	}
 
-	class = lock->class[subclass];
-	/* not cached yet? */
+	if (!subclass)
+		class = lock->class_cache;
+	/*
+	 * Not cached yet or subclass?
+	 */
 	if (unlikely(!class)) {
 		class = register_lock_class(lock, subclass);
 		if (!class)
@@ -2449,48 +2474,44 @@ void lockdep_free_key_range(void *start, unsigned long size)
 
 void lockdep_reset_lock(struct lockdep_map *lock)
 {
-	struct lock_class *class, *next, *entry;
+	struct lock_class *class, *next;
 	struct list_head *head;
 	unsigned long flags;
 	int i, j;
 
 	raw_local_irq_save(flags);
-	__raw_spin_lock(&hash_lock);
 
 	/*
-	 * Remove all classes this lock has:
+	 * Remove all classes this lock might have:
+	 */
+	for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
+		/*
+		 * If the class exists we look it up and zap it:
+		 */
+		class = look_up_lock_class(lock, j);
+		if (class)
+			zap_class(class);
+	}
+	/*
+	 * Debug check: in the end all mapped classes should
+	 * be gone.
 	 */
+	__raw_spin_lock(&hash_lock);
 	for (i = 0; i < CLASSHASH_SIZE; i++) {
 		head = classhash_table + i;
 		if (list_empty(head))
 			continue;
 		list_for_each_entry_safe(class, next, head, hash_entry) {
-			for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
-				entry = lock->class[j];
-				if (class == entry) {
-					zap_class(class);
-					lock->class[j] = NULL;
-					break;
-				}
+			if (unlikely(class == lock->class_cache)) {
+				__raw_spin_unlock(&hash_lock);
+				DEBUG_LOCKS_WARN_ON(1);
+				goto out_restore;
 			}
 		}
 	}
-
-	/*
-	 * Debug check: in the end all mapped classes should
-	 * be gone.
-	 */
-	for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
-		entry = lock->class[j];
-		if (!entry)
-			continue;
-		__raw_spin_unlock(&hash_lock);
-		DEBUG_LOCKS_WARN_ON(1);
-		raw_local_irq_restore(flags);
-		return;
-	}
-
 	__raw_spin_unlock(&hash_lock);
+
+out_restore:
 	raw_local_irq_restore(flags);
 }