From ae2a823643d71f40751259266f7c2e7d90909662 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Thu, 11 Aug 2022 15:29:46 -0700 Subject: dcache: move the DCACHE_OP_COMPARE case out of the __d_lookup_rcu loop __d_lookup_rcu() is one of the hottest functions in the kernel on certain loads, and it is complicated by filesystems that might want to have their own name compare function. We can improve code generation by moving the test of DCACHE_OP_COMPARE outside the loop, which makes the loop itself much simpler, at the cost of some code duplication. But both cases end up being simpler, and the "native" direct case-sensitive compare particularly so. Cc: Al Viro Signed-off-by: Linus Torvalds --- fs/dcache.c | 72 +++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 23 deletions(-) (limited to 'fs/dcache.c') diff --git a/fs/dcache.c b/fs/dcache.c index c5dc32a59c76..bb0c4d0038db 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -2270,6 +2270,48 @@ bool d_same_name(const struct dentry *dentry, const struct dentry *parent, } EXPORT_SYMBOL_GPL(d_same_name); +/* + * This is __d_lookup_rcu() when the parent dentry has + * DCACHE_OP_COMPARE, which makes things much nastier. + */ +static noinline struct dentry *__d_lookup_rcu_op_compare( + const struct dentry *parent, + const struct qstr *name, + unsigned *seqp) +{ + u64 hashlen = name->hash_len; + struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen)); + struct hlist_bl_node *node; + struct dentry *dentry; + + hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { + int tlen; + const char *tname; + unsigned seq; + +seqretry: + seq = raw_seqcount_begin(&dentry->d_seq); + if (dentry->d_parent != parent) + continue; + if (d_unhashed(dentry)) + continue; + if (dentry->d_name.hash != hashlen_hash(hashlen)) + continue; + tlen = dentry->d_name.len; + tname = dentry->d_name.name; + /* we want a consistent (name,len) pair */ + if (read_seqcount_retry(&dentry->d_seq, seq)) { + cpu_relax(); + goto seqretry; + } + if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0) + continue; + *seqp = seq; + return dentry; + } + return NULL; +} + /** * __d_lookup_rcu - search for a dentry (racy, store-free) * @parent: parent dentry @@ -2316,6 +2358,9 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, * Keep the two functions in sync. */ + if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) + return __d_lookup_rcu_op_compare(parent, name, seqp); + /* * The hash list is protected using RCU. * @@ -2332,7 +2377,6 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { unsigned seq; -seqretry: /* * The dentry sequence count protects us from concurrent * renames, and thus protects parent and name fields. @@ -2355,28 +2399,10 @@ seqretry: continue; if (d_unhashed(dentry)) continue; - - if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { - int tlen; - const char *tname; - if (dentry->d_name.hash != hashlen_hash(hashlen)) - continue; - tlen = dentry->d_name.len; - tname = dentry->d_name.name; - /* we want a consistent (name,len) pair */ - if (read_seqcount_retry(&dentry->d_seq, seq)) { - cpu_relax(); - goto seqretry; - } - if (parent->d_op->d_compare(dentry, - tlen, tname, name) != 0) - continue; - } else { - if (dentry->d_name.hash_len != hashlen) - continue; - if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) - continue; - } + if (dentry->d_name.hash_len != hashlen) + continue; + if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) + continue; *seqp = seq; return dentry; } -- cgit v1.2.3