[linux-block.git] / fs / bcachefs / ec.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EC_H
#define _BCACHEFS_EC_H

#include "ec_types.h"
#include "buckets_types.h"
#include "extents_types.h"

enum bkey_invalid_flags;

int bch2_stripe_invalid(struct bch_fs *, struct bkey_s_c,
			enum bkey_invalid_flags, struct printbuf *);
void bch2_stripe_to_text(struct printbuf *, struct bch_fs *,
			 struct bkey_s_c);

#define bch2_bkey_ops_stripe ((struct bkey_ops) {	\
	.key_invalid	= bch2_stripe_invalid,		\
	.val_to_text	= bch2_stripe_to_text,		\
	.swab		= bch2_ptr_swab,		\
	.trans_trigger	= bch2_trans_mark_stripe,	\
	.atomic_trigger	= bch2_mark_stripe,		\
	.min_val_size	= 8,				\
})

static inline unsigned stripe_csums_per_device(const struct bch_stripe *s)
{
	return DIV_ROUND_UP(le16_to_cpu(s->sectors),
			    1 << s->csum_granularity_bits);
}

static inline unsigned stripe_csum_offset(const struct bch_stripe *s,
					  unsigned dev, unsigned csum_idx)
{
	unsigned csum_bytes = bch_crc_bytes[s->csum_type];

	return sizeof(struct bch_stripe) +
		sizeof(struct bch_extent_ptr) * s->nr_blocks +
		(dev * stripe_csums_per_device(s) + csum_idx) * csum_bytes;
}

static inline unsigned stripe_blockcount_offset(const struct bch_stripe *s,
						unsigned idx)
{
	return stripe_csum_offset(s, s->nr_blocks, 0) +
		sizeof(u16) * idx;
}

static inline unsigned stripe_blockcount_get(const struct bch_stripe *s,
					     unsigned idx)
{
	return le16_to_cpup((void *) s + stripe_blockcount_offset(s, idx));
}

static inline void stripe_blockcount_set(struct bch_stripe *s,
					 unsigned idx, unsigned v)
{
	__le16 *p = (void *) s + stripe_blockcount_offset(s, idx);

	*p = cpu_to_le16(v);
}

static inline unsigned stripe_val_u64s(const struct bch_stripe *s)
{
	return DIV_ROUND_UP(stripe_blockcount_offset(s, s->nr_blocks),
			    sizeof(u64));
}

static inline void *stripe_csum(struct bch_stripe *s,
				unsigned block, unsigned csum_idx)
{
	EBUG_ON(block >= s->nr_blocks);
	EBUG_ON(csum_idx >= stripe_csums_per_device(s));

	return (void *) s + stripe_csum_offset(s, block, csum_idx);
}

static inline struct bch_csum stripe_csum_get(struct bch_stripe *s,
				   unsigned block, unsigned csum_idx)
{
	struct bch_csum csum = { 0 };

	memcpy(&csum, stripe_csum(s, block, csum_idx), bch_crc_bytes[s->csum_type]);
	return csum;
}

static inline void stripe_csum_set(struct bch_stripe *s,
				   unsigned block, unsigned csum_idx,
				   struct bch_csum csum)
{
	memcpy(stripe_csum(s, block, csum_idx), &csum, bch_crc_bytes[s->csum_type]);
}

static inline bool __bch2_ptr_matches_stripe(const struct bch_extent_ptr *stripe_ptr,
					     const struct bch_extent_ptr *data_ptr,
					     unsigned sectors)
{
	return  data_ptr->dev    == stripe_ptr->dev &&
		data_ptr->gen    == stripe_ptr->gen &&
		data_ptr->offset >= stripe_ptr->offset &&
		data_ptr->offset  < stripe_ptr->offset + sectors;
}

static inline bool bch2_ptr_matches_stripe(const struct bch_stripe *s,
					   struct extent_ptr_decoded p)
{
	unsigned nr_data = s->nr_blocks - s->nr_redundant;

	BUG_ON(!p.has_ec);

	if (p.ec.block >= nr_data)
		return false;

	return __bch2_ptr_matches_stripe(&s->ptrs[p.ec.block], &p.ptr,
					 le16_to_cpu(s->sectors));
}

static inline bool bch2_ptr_matches_stripe_m(const struct gc_stripe *m,
					     struct extent_ptr_decoded p)
{
	unsigned nr_data = m->nr_blocks - m->nr_redundant;

	BUG_ON(!p.has_ec);

	if (p.ec.block >= nr_data)
		return false;

	return __bch2_ptr_matches_stripe(&m->ptrs[p.ec.block], &p.ptr,
					 m->sectors);
}

struct bch_read_bio;

struct ec_stripe_buf {
	/* might not be buffering the entire stripe: */
	unsigned		offset;
	unsigned		size;
	unsigned long		valid[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];

	void			*data[BCH_BKEY_PTRS_MAX];

	__BKEY_PADDED(key, 255);
};

struct ec_stripe_head;

enum ec_stripe_ref {
	STRIPE_REF_io,
	STRIPE_REF_stripe,
	STRIPE_REF_NR
};

struct ec_stripe_new {
	struct bch_fs		*c;
	struct ec_stripe_head	*h;
	struct mutex		lock;
	struct list_head	list;

	struct hlist_node	hash;
	u64			idx;

	struct closure		iodone;

	atomic_t		ref[STRIPE_REF_NR];

	int			err;

	u8			nr_data;
	u8			nr_parity;
	bool			allocated;
	bool			pending;
	bool			have_existing_stripe;

	unsigned long		blocks_gotten[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
	unsigned long		blocks_allocated[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
	open_bucket_idx_t	blocks[BCH_BKEY_PTRS_MAX];
	struct disk_reservation	res;

	struct ec_stripe_buf	new_stripe;
	struct ec_stripe_buf	existing_stripe;
};

struct ec_stripe_head {
	struct list_head	list;
	struct mutex		lock;

	unsigned		target;
	unsigned		algo;
	unsigned		redundancy;
	enum bch_watermark	watermark;

	struct bch_devs_mask	devs;
	unsigned		nr_active_devs;

	unsigned		blocksize;

	struct dev_stripe_state	block_stripe;
	struct dev_stripe_state	parity_stripe;

	struct ec_stripe_new	*s;
};

int bch2_ec_read_extent(struct btree_trans *, struct bch_read_bio *);

void *bch2_writepoint_ec_buf(struct bch_fs *, struct write_point *);

void bch2_ec_bucket_cancel(struct bch_fs *, struct open_bucket *);

int bch2_ec_stripe_new_alloc(struct bch_fs *, struct ec_stripe_head *);

void bch2_ec_stripe_head_put(struct bch_fs *, struct ec_stripe_head *);
struct ec_stripe_head *bch2_ec_stripe_head_get(struct btree_trans *,
			unsigned, unsigned, unsigned,
			enum bch_watermark, struct closure *);

void bch2_stripes_heap_update(struct bch_fs *, struct stripe *, size_t);
void bch2_stripes_heap_del(struct bch_fs *, struct stripe *, size_t);
void bch2_stripes_heap_insert(struct bch_fs *, struct stripe *, size_t);

void bch2_do_stripe_deletes(struct bch_fs *);
void bch2_ec_do_stripe_creates(struct bch_fs *);
void bch2_ec_stripe_new_free(struct bch_fs *, struct ec_stripe_new *);

static inline void ec_stripe_new_get(struct ec_stripe_new *s,
				     enum ec_stripe_ref ref)
{
	atomic_inc(&s->ref[ref]);
}

static inline void ec_stripe_new_put(struct bch_fs *c, struct ec_stripe_new *s,
				     enum ec_stripe_ref ref)
{
	BUG_ON(atomic_read(&s->ref[ref]) <= 0);

	if (atomic_dec_and_test(&s->ref[ref]))
		switch (ref) {
		case STRIPE_REF_stripe:
			bch2_ec_stripe_new_free(c, s);
			break;
		case STRIPE_REF_io:
			bch2_ec_do_stripe_creates(c);
			break;
		default:
			BUG();
		}
}

void bch2_ec_stop_dev(struct bch_fs *, struct bch_dev *);
void bch2_fs_ec_stop(struct bch_fs *);
void bch2_fs_ec_flush(struct bch_fs *);

int bch2_stripes_read(struct bch_fs *);

void bch2_stripes_heap_to_text(struct printbuf *, struct bch_fs *);
void bch2_new_stripes_to_text(struct printbuf *, struct bch_fs *);

void bch2_fs_ec_exit(struct bch_fs *);
void bch2_fs_ec_init_early(struct bch_fs *);
int bch2_fs_ec_init(struct bch_fs *);

#endif /* _BCACHEFS_EC_H */
Commit	Line	Data
cd575ddf KO	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _BCACHEFS_EC_H
	3	#define _BCACHEFS_EC_H
	4
	5	#include "ec_types.h"
af4d05c4	6	#include "buckets_types.h"
c501fef6	7	#include "extents_types.h"
cd575ddf	8
8726dc93 KO	9	enum bkey_invalid_flags;
8726dc93 KO	10
b65db750	11	int bch2_stripe_invalid(struct bch_fs *, struct bkey_s_c,
8726dc93	12	enum bkey_invalid_flags, struct printbuf *);
26609b61	13	void bch2_stripe_to_text(struct printbuf , struct bch_fs ,
cd575ddf KO	14	struct bkey_s_c);
cd575ddf KO	15
a1019576	16	#define bch2_bkey_ops_stripe ((struct bkey_ops) { \
26609b61 KO	17	.key_invalid = bch2_stripe_invalid, \
26609b61 KO	18	.val_to_text = bch2_stripe_to_text, \
1f49dafc	19	.swab = bch2_ptr_swab, \
880e2275 KO	20	.trans_trigger = bch2_trans_mark_stripe, \
880e2275 KO	21	.atomic_trigger = bch2_mark_stripe, \
174f930b	22	.min_val_size = 8, \
a1019576	23	})
cd575ddf	24
61c8d7c8 KO	25	static inline unsigned stripe_csums_per_device(const struct bch_stripe *s)
	26	{
	27	return DIV_ROUND_UP(le16_to_cpu(s->sectors),
	28	1 << s->csum_granularity_bits);
	29	}
	30
	31	static inline unsigned stripe_csum_offset(const struct bch_stripe *s,
	32	unsigned dev, unsigned csum_idx)
	33	{
	34	unsigned csum_bytes = bch_crc_bytes[s->csum_type];
	35
	36	return sizeof(struct bch_stripe) +
	37	sizeof(struct bch_extent_ptr) * s->nr_blocks +
	38	(dev * stripe_csums_per_device(s) + csum_idx) * csum_bytes;
	39	}
	40
	41	static inline unsigned stripe_blockcount_offset(const struct bch_stripe *s,
	42	unsigned idx)
	43	{
	44	return stripe_csum_offset(s, s->nr_blocks, 0) +
	45	sizeof(u16) * idx;
	46	}
	47
	48	static inline unsigned stripe_blockcount_get(const struct bch_stripe *s,
	49	unsigned idx)
	50	{
	51	return le16_to_cpup((void *) s + stripe_blockcount_offset(s, idx));
	52	}
	53
	54	static inline void stripe_blockcount_set(struct bch_stripe *s,
	55	unsigned idx, unsigned v)
	56	{
	57	__le16 p = (void ) s + stripe_blockcount_offset(s, idx);
	58
	59	*p = cpu_to_le16(v);
	60	}
	61
	62	static inline unsigned stripe_val_u64s(const struct bch_stripe *s)
	63	{
	64	return DIV_ROUND_UP(stripe_blockcount_offset(s, s->nr_blocks),
	65	sizeof(u64));
	66	}
	67
	68	static inline void stripe_csum(struct bch_stripe s,
2a3731e3	69	unsigned block, unsigned csum_idx)
61c8d7c8	70	{
2a3731e3 KO	71	EBUG_ON(block >= s->nr_blocks);
	72	EBUG_ON(csum_idx >= stripe_csums_per_device(s));
	73
	74	return (void *) s + stripe_csum_offset(s, block, csum_idx);
	75	}
	76
	77	static inline struct bch_csum stripe_csum_get(struct bch_stripe *s,
	78	unsigned block, unsigned csum_idx)
	79	{
	80	struct bch_csum csum = { 0 };
	81
	82	memcpy(&csum, stripe_csum(s, block, csum_idx), bch_crc_bytes[s->csum_type]);
	83	return csum;
	84	}
	85
	86	static inline void stripe_csum_set(struct bch_stripe *s,
	87	unsigned block, unsigned csum_idx,
	88	struct bch_csum csum)
	89	{
	90	memcpy(stripe_csum(s, block, csum_idx), &csum, bch_crc_bytes[s->csum_type]);
	91	}
	92
b3b66e30 KO	93	static inline bool __bch2_ptr_matches_stripe(const struct bch_extent_ptr *stripe_ptr,
	94	const struct bch_extent_ptr *data_ptr,
	95	unsigned sectors)
	96	{
	97	return data_ptr->dev == stripe_ptr->dev &&
	98	data_ptr->gen == stripe_ptr->gen &&
	99	data_ptr->offset >= stripe_ptr->offset &&
	100	data_ptr->offset < stripe_ptr->offset + sectors;
	101	}
	102
	103	static inline bool bch2_ptr_matches_stripe(const struct bch_stripe *s,
	104	struct extent_ptr_decoded p)
2a3731e3 KO	105	{
	106	unsigned nr_data = s->nr_blocks - s->nr_redundant;
	107
b3b66e30 KO	108	BUG_ON(!p.has_ec);
	109
	110	if (p.ec.block >= nr_data)
2a3731e3 KO	111	return false;
2a3731e3 KO	112
b3b66e30 KO	113	return __bch2_ptr_matches_stripe(&s->ptrs[p.ec.block], &p.ptr,
b3b66e30 KO	114	le16_to_cpu(s->sectors));
2a3731e3 KO	115	}
2a3731e3 KO	116
990d42d1	117	static inline bool bch2_ptr_matches_stripe_m(const struct gc_stripe *m,
b3b66e30	118	struct extent_ptr_decoded p)
2a3731e3	119	{
b3b66e30 KO	120	unsigned nr_data = m->nr_blocks - m->nr_redundant;
b3b66e30 KO	121
2a3731e3 KO	122	BUG_ON(!p.has_ec);
2a3731e3 KO	123
b3b66e30 KO	124	if (p.ec.block >= nr_data)
	125	return false;
	126
	127	return __bch2_ptr_matches_stripe(&m->ptrs[p.ec.block], &p.ptr,
	128	m->sectors);
61c8d7c8 KO	129	}
61c8d7c8 KO	130
cd575ddf KO	131	struct bch_read_bio;
	132
	133	struct ec_stripe_buf {
	134	/* might not be buffering the entire stripe: */
	135	unsigned offset;
	136	unsigned size;
ffb7c3d3	137	unsigned long valid[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
cd575ddf	138
ffb7c3d3	139	void *data[BCH_BKEY_PTRS_MAX];
cd575ddf	140
bf5a261c	141	__BKEY_PADDED(key, 255);
cd575ddf KO	142	};
	143
	144	struct ec_stripe_head;
	145
fba053d2 KO	146	enum ec_stripe_ref {
	147	STRIPE_REF_io,
	148	STRIPE_REF_stripe,
	149	STRIPE_REF_NR
	150	};
	151
cd575ddf KO	152	struct ec_stripe_new {
	153	struct bch_fs *c;
	154	struct ec_stripe_head *h;
	155	struct mutex lock;
	156	struct list_head list;
4b1e6699 KO	157
	158	struct hlist_node hash;
	159	u64 idx;
	160
81d8599e	161	struct closure iodone;
cd575ddf	162
fba053d2	163	atomic_t ref[STRIPE_REF_NR];
cd575ddf KO	164
	165	int err;
	166
f6b94a3b KO	167	u8 nr_data;
	168	u8 nr_parity;
	169	bool allocated;
	170	bool pending;
81d8599e	171	bool have_existing_stripe;
0ba95acc	172
6c7585b0	173	unsigned long blocks_gotten[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
ffb7c3d3	174	unsigned long blocks_allocated[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
6c7585b0	175	open_bucket_idx_t blocks[BCH_BKEY_PTRS_MAX];
af4d05c4	176	struct disk_reservation res;
cd575ddf	177
81d8599e KO	178	struct ec_stripe_buf new_stripe;
81d8599e KO	179	struct ec_stripe_buf existing_stripe;
cd575ddf KO	180	};
	181
	182	struct ec_stripe_head {
	183	struct list_head list;
	184	struct mutex lock;
	185
cd575ddf KO	186	unsigned target;
	187	unsigned algo;
	188	unsigned redundancy;
e53a961c	189	enum bch_watermark watermark;
cd575ddf KO	190
	191	struct bch_devs_mask devs;
	192	unsigned nr_active_devs;
	193
	194	unsigned blocksize;
	195
	196	struct dev_stripe_state block_stripe;
	197	struct dev_stripe_state parity_stripe;
	198
cd575ddf KO	199	struct ec_stripe_new *s;
	200	};
	201
aa982665	202	int bch2_ec_read_extent(struct btree_trans , struct bch_read_bio );
cd575ddf KO	203
cd575ddf KO	204	void bch2_writepoint_ec_buf(struct bch_fs , struct write_point *);
cd575ddf	205
cd575ddf KO	206	void bch2_ec_bucket_cancel(struct bch_fs , struct open_bucket );
	207
	208	int bch2_ec_stripe_new_alloc(struct bch_fs , struct ec_stripe_head );
	209
703e2a43	210	void bch2_ec_stripe_head_put(struct bch_fs , struct ec_stripe_head );
2c7dd446	211	struct ec_stripe_head bch2_ec_stripe_head_get(struct btree_trans ,
a1fb08f5	212	unsigned, unsigned, unsigned,
e53a961c	213	enum bch_watermark, struct closure *);
cd575ddf	214
dfe9bfb3 KO	215	void bch2_stripes_heap_update(struct bch_fs , struct stripe , size_t);
	216	void bch2_stripes_heap_del(struct bch_fs , struct stripe , size_t);
	217	void bch2_stripes_heap_insert(struct bch_fs , struct stripe , size_t);
cd575ddf	218
dd81a060	219	void bch2_do_stripe_deletes(struct bch_fs *);
2a912a9a	220	void bch2_ec_do_stripe_creates(struct bch_fs *);
fba053d2	221	void bch2_ec_stripe_new_free(struct bch_fs , struct ec_stripe_new );
2a912a9a	222
fba053d2 KO	223	static inline void ec_stripe_new_get(struct ec_stripe_new *s,
fba053d2 KO	224	enum ec_stripe_ref ref)
2a912a9a	225	{
fba053d2	226	atomic_inc(&s->ref[ref]);
2a912a9a KO	227	}
2a912a9a KO	228
fba053d2 KO	229	static inline void ec_stripe_new_put(struct bch_fs c, struct ec_stripe_new s,
fba053d2 KO	230	enum ec_stripe_ref ref)
2a912a9a	231	{
fba053d2 KO	232	BUG_ON(atomic_read(&s->ref[ref]) <= 0);
	233
	234	if (atomic_dec_and_test(&s->ref[ref]))
	235	switch (ref) {
	236	case STRIPE_REF_stripe:
	237	bch2_ec_stripe_new_free(c, s);
	238	break;
	239	case STRIPE_REF_io:
	240	bch2_ec_do_stripe_creates(c);
	241	break;
	242	default:
439c172b	243	BUG();
fba053d2	244	}
2a912a9a	245	}
dd81a060	246
cd575ddf	247	void bch2_ec_stop_dev(struct bch_fs , struct bch_dev );
b40901b0 KO	248	void bch2_fs_ec_stop(struct bch_fs *);
b40901b0 KO	249	void bch2_fs_ec_flush(struct bch_fs *);
cd575ddf	250
ac1019d3	251	int bch2_stripes_read(struct bch_fs *);
61c8d7c8	252
ba6dd1dd	253	void bch2_stripes_heap_to_text(struct printbuf , struct bch_fs );
7807e143	254	void bch2_new_stripes_to_text(struct printbuf , struct bch_fs );
ba6dd1dd	255
cd575ddf	256	void bch2_fs_ec_exit(struct bch_fs *);
84c72755	257	void bch2_fs_ec_init_early(struct bch_fs *);
cd575ddf KO	258	int bch2_fs_ec_init(struct bch_fs *);
	259
	260	#endif /* _BCACHEFS_EC_H */