bcachefs: Inline make_bfloat() into __build_ro_aux_tree()

This is a fast path - also, lift out the checks/init for min/max key.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

1 files changed, 46 insertions, 48 deletions

diff --git a/fs/bcachefs/bset.c b/fs/bcachefs/bset.c
index fc8a7cc2f4bd..89d511db2c50 100644
--- a/fs/bcachefs/bset.c
+++ b/fs/bcachefs/bset.c
@@ -592,53 +592,23 @@ static inline unsigned bkey_mantissa(const struct bkey_packed *k,
 	return (u16) v;
 }
 
-static void make_bfloat(struct btree *b, struct bset_tree *t,
-			unsigned j,
-			struct bkey_packed *min_key,
-			struct bkey_packed *max_key)
+__always_inline
+static inline void __make_bfloat(struct btree *b, struct bset_tree *t,
+				 unsigned j,
+				 struct bkey_packed *min_key,
+				 struct bkey_packed *max_key)
 {
 	struct bkey_float *f = bkey_float(b, t, j);
 	struct bkey_packed *m = tree_to_bkey(b, t, j);
-	struct bkey_packed *l, *r;
+	struct bkey_packed *l = is_power_of_2(j)
+		? min_key
+		: tree_to_prev_bkey(b, t, j >> ffs(j));
+	struct bkey_packed *r = is_power_of_2(j + 1)
+		? max_key
+		: tree_to_bkey(b, t, j >> (ffz(j) + 1));
 	unsigned mantissa;
 	int shift, exponent, high_bit;
 
-	if (is_power_of_2(j)) {
-		l = min_key;
-
-		if (!l->u64s) {
-			if (!bkey_pack_pos(l, b->data->min_key, b)) {
-				struct bkey_i tmp;
-
-				bkey_init(&tmp.k);
-				tmp.k.p = b->data->min_key;
-				bkey_copy(l, &tmp);
-			}
-		}
-	} else {
-		l = tree_to_prev_bkey(b, t, j >> ffs(j));
-
-		EBUG_ON(m < l);
-	}
-
-	if (is_power_of_2(j + 1)) {
-		r = max_key;
-
-		if (!r->u64s) {
-			if (!bkey_pack_pos(r, t->max_key, b)) {
-				struct bkey_i tmp;
-
-				bkey_init(&tmp.k);
-				tmp.k.p = t->max_key;
-				bkey_copy(r, &tmp);
-			}
-		}
-	} else {
-		r = tree_to_bkey(b, t, j >> (ffz(j) + 1));
-
-		EBUG_ON(m > r);
-	}
-
 	/*
 	 * for failed bfloats, the lookup code falls back to comparing against
 	 * the original key.
@@ -695,6 +665,30 @@ static void make_bfloat(struct btree *b, struct bset_tree *t,
 	f->mantissa = mantissa;
 }
 
+static void make_bfloat(struct btree *b, struct bset_tree *t,
+			unsigned j,
+			struct bkey_packed *min_key,
+			struct bkey_packed *max_key)
+{
+	struct bkey_i *k;
+
+	if (is_power_of_2(j) &&
+	    !min_key->u64s) {
+		k = (void *) min_key;
+		bkey_init(&k->k);
+		k->k.p = b->data->min_key;
+	}
+
+	if (is_power_of_2(j + 1) &&
+	    !max_key->u64s) {
+		k = (void *) max_key;
+		bkey_init(&k->k);
+		k->k.p = t->max_key;
+	}
+
+	__make_bfloat(b, t, j, min_key, max_key);
+}
+
 /* bytes remaining - only valid for last bset: */
 static unsigned __bset_tree_capacity(const struct btree *b, const struct bset_tree *t)
 {
@@ -714,7 +708,7 @@ static unsigned bset_rw_tree_capacity(const struct btree *b, const struct bset_t
 	return __bset_tree_capacity(b, t) / sizeof(struct rw_aux_tree);
 }
 
-static void __build_rw_aux_tree(struct btree *b, struct bset_tree *t)
+static noinline void __build_rw_aux_tree(struct btree *b, struct bset_tree *t)
 {
 	struct bkey_packed *k;
 
@@ -733,15 +727,12 @@ static void __build_rw_aux_tree(struct btree *b, struct bset_tree *t)
 	}
 }
 
-static void __build_ro_aux_tree(struct btree *b, struct bset_tree *t)
+static noinline void __build_ro_aux_tree(struct btree *b, struct bset_tree *t)
 {
 	struct bkey_packed *prev = NULL, *k = btree_bkey_first(b, t);
-	struct bkey_packed min_key, max_key;
+	struct bkey_i min_key, max_key;
 	unsigned j, cacheline = 1;
 
-	/* signal to make_bfloat() that they're uninitialized: */
-	min_key.u64s = max_key.u64s = 0;
-
 	t->size = min(bkey_to_cacheline(b, t, btree_bkey_last(b, t)),
 		      bset_ro_tree_capacity(b, t));
 retry:
@@ -777,9 +768,16 @@ retry:
 
 	t->max_key = bkey_unpack_pos(b, prev);
 
+	bkey_init(&min_key.k);
+	min_key.k.p = b->data->min_key;
+	bkey_init(&max_key.k);
+	max_key.k.p = t->max_key;
+
 	/* Then we build the tree */
 	eytzinger1_for_each(j, t->size)
-		make_bfloat(b, t, j, &min_key, &max_key);
+		__make_bfloat(b, t, j,
+			      bkey_to_packed(&min_key),
+			      bkey_to_packed(&max_key));
 }
 
 static void bset_alloc_tree(struct btree *b, struct bset_tree *t)