[linux-block.git] / fs / bcachefs / ec.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EC_H
#define _BCACHEFS_EC_H

#include "ec_types.h"
#include "buckets_types.h"
#include "extents_types.h"

int bch2_stripe_invalid(const struct bch_fs *, struct bkey_s_c,
			unsigned, struct printbuf *);
void bch2_stripe_to_text(struct printbuf *, struct bch_fs *,
			 struct bkey_s_c);

#define bch2_bkey_ops_stripe ((struct bkey_ops) {	\
	.key_invalid	= bch2_stripe_invalid,		\
	.val_to_text	= bch2_stripe_to_text,		\
	.swab		= bch2_ptr_swab,		\
	.trans_trigger	= bch2_trans_mark_stripe,	\
	.atomic_trigger	= bch2_mark_stripe,		\
})

static inline unsigned stripe_csums_per_device(const struct bch_stripe *s)
{
	return DIV_ROUND_UP(le16_to_cpu(s->sectors),
			    1 << s->csum_granularity_bits);
}

static inline unsigned stripe_csum_offset(const struct bch_stripe *s,
					  unsigned dev, unsigned csum_idx)
{
	unsigned csum_bytes = bch_crc_bytes[s->csum_type];

	return sizeof(struct bch_stripe) +
		sizeof(struct bch_extent_ptr) * s->nr_blocks +
		(dev * stripe_csums_per_device(s) + csum_idx) * csum_bytes;
}

static inline unsigned stripe_blockcount_offset(const struct bch_stripe *s,
						unsigned idx)
{
	return stripe_csum_offset(s, s->nr_blocks, 0) +
		sizeof(u16) * idx;
}

static inline unsigned stripe_blockcount_get(const struct bch_stripe *s,
					     unsigned idx)
{
	return le16_to_cpup((void *) s + stripe_blockcount_offset(s, idx));
}

static inline void stripe_blockcount_set(struct bch_stripe *s,
					 unsigned idx, unsigned v)
{
	__le16 *p = (void *) s + stripe_blockcount_offset(s, idx);

	*p = cpu_to_le16(v);
}

static inline unsigned stripe_val_u64s(const struct bch_stripe *s)
{
	return DIV_ROUND_UP(stripe_blockcount_offset(s, s->nr_blocks),
			    sizeof(u64));
}

static inline void *stripe_csum(struct bch_stripe *s,
				unsigned block, unsigned csum_idx)
{
	EBUG_ON(block >= s->nr_blocks);
	EBUG_ON(csum_idx >= stripe_csums_per_device(s));

	return (void *) s + stripe_csum_offset(s, block, csum_idx);
}

static inline struct bch_csum stripe_csum_get(struct bch_stripe *s,
				   unsigned block, unsigned csum_idx)
{
	struct bch_csum csum = { 0 };

	memcpy(&csum, stripe_csum(s, block, csum_idx), bch_crc_bytes[s->csum_type]);
	return csum;
}

static inline void stripe_csum_set(struct bch_stripe *s,
				   unsigned block, unsigned csum_idx,
				   struct bch_csum csum)
{
	memcpy(stripe_csum(s, block, csum_idx), &csum, bch_crc_bytes[s->csum_type]);
}

static inline bool __bch2_ptr_matches_stripe(const struct bch_extent_ptr *stripe_ptr,
					     const struct bch_extent_ptr *data_ptr,
					     unsigned sectors)
{
	return  data_ptr->dev    == stripe_ptr->dev &&
		data_ptr->gen    == stripe_ptr->gen &&
		data_ptr->offset >= stripe_ptr->offset &&
		data_ptr->offset  < stripe_ptr->offset + sectors;
}

static inline bool bch2_ptr_matches_stripe(const struct bch_stripe *s,
					   struct extent_ptr_decoded p)
{
	unsigned nr_data = s->nr_blocks - s->nr_redundant;

	BUG_ON(!p.has_ec);

	if (p.ec.block >= nr_data)
		return false;

	return __bch2_ptr_matches_stripe(&s->ptrs[p.ec.block], &p.ptr,
					 le16_to_cpu(s->sectors));
}

static inline bool bch2_ptr_matches_stripe_m(const struct gc_stripe *m,
					     struct extent_ptr_decoded p)
{
	unsigned nr_data = m->nr_blocks - m->nr_redundant;

	BUG_ON(!p.has_ec);

	if (p.ec.block >= nr_data)
		return false;

	return __bch2_ptr_matches_stripe(&m->ptrs[p.ec.block], &p.ptr,
					 m->sectors);
}

struct bch_read_bio;

struct ec_stripe_buf {
	/* might not be buffering the entire stripe: */
	unsigned		offset;
	unsigned		size;
	unsigned long		valid[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];

	void			*data[BCH_BKEY_PTRS_MAX];

	union {
		struct bkey_i_stripe	key;
		u64			pad[255];
	};
};

struct ec_stripe_head;

struct ec_stripe_new {
	struct bch_fs		*c;
	struct ec_stripe_head	*h;
	struct mutex		lock;
	struct list_head	list;
	struct closure		iodone;

	/* counts in flight writes, stripe is created when pin == 0 */
	atomic_t		pin;

	int			err;

	u8			nr_data;
	u8			nr_parity;
	bool			allocated;
	bool			pending;
	bool			have_existing_stripe;

	unsigned long		blocks_gotten[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
	unsigned long		blocks_allocated[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
	open_bucket_idx_t	blocks[BCH_BKEY_PTRS_MAX];
	struct disk_reservation	res;

	struct ec_stripe_buf	new_stripe;
	struct ec_stripe_buf	existing_stripe;
};

struct ec_stripe_head {
	struct list_head	list;
	struct mutex		lock;

	unsigned		target;
	unsigned		algo;
	unsigned		redundancy;
	bool			copygc;

	struct bch_devs_mask	devs;
	unsigned		nr_active_devs;

	unsigned		blocksize;

	struct dev_stripe_state	block_stripe;
	struct dev_stripe_state	parity_stripe;

	struct ec_stripe_new	*s;
};

int bch2_ec_read_extent(struct bch_fs *, struct bch_read_bio *);

void *bch2_writepoint_ec_buf(struct bch_fs *, struct write_point *);

void bch2_ec_bucket_written(struct bch_fs *, struct open_bucket *);
void bch2_ec_bucket_cancel(struct bch_fs *, struct open_bucket *);

int bch2_ec_stripe_new_alloc(struct bch_fs *, struct ec_stripe_head *);

void bch2_ec_stripe_head_put(struct bch_fs *, struct ec_stripe_head *);
struct ec_stripe_head *bch2_ec_stripe_head_get(struct bch_fs *,
			unsigned, unsigned, unsigned, bool, struct closure *);

void bch2_stripes_heap_update(struct bch_fs *, struct stripe *, size_t);
void bch2_stripes_heap_del(struct bch_fs *, struct stripe *, size_t);
void bch2_stripes_heap_insert(struct bch_fs *, struct stripe *, size_t);

void bch2_do_stripe_deletes(struct bch_fs *);

void bch2_ec_stop_dev(struct bch_fs *, struct bch_dev *);

void bch2_ec_flush_new_stripes(struct bch_fs *);

void bch2_stripes_heap_start(struct bch_fs *);

int bch2_stripes_read(struct bch_fs *);

void bch2_stripes_heap_to_text(struct printbuf *, struct bch_fs *);
void bch2_new_stripes_to_text(struct printbuf *, struct bch_fs *);

void bch2_fs_ec_exit(struct bch_fs *);
void bch2_fs_ec_init_early(struct bch_fs *);
int bch2_fs_ec_init(struct bch_fs *);

#endif /* _BCACHEFS_EC_H */
Commit	Line	Data
cd575ddf KO	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _BCACHEFS_EC_H
	3	#define _BCACHEFS_EC_H
	4
	5	#include "ec_types.h"
af4d05c4	6	#include "buckets_types.h"
c501fef6	7	#include "extents_types.h"
cd575ddf	8
f0ac7df2	9	int bch2_stripe_invalid(const struct bch_fs *, struct bkey_s_c,
facafdcb	10	unsigned, struct printbuf *);
26609b61	11	void bch2_stripe_to_text(struct printbuf , struct bch_fs ,
cd575ddf KO	12	struct bkey_s_c);
cd575ddf KO	13
a1019576	14	#define bch2_bkey_ops_stripe ((struct bkey_ops) { \
26609b61 KO	15	.key_invalid = bch2_stripe_invalid, \
26609b61 KO	16	.val_to_text = bch2_stripe_to_text, \
1f49dafc	17	.swab = bch2_ptr_swab, \
880e2275 KO	18	.trans_trigger = bch2_trans_mark_stripe, \
880e2275 KO	19	.atomic_trigger = bch2_mark_stripe, \
a1019576	20	})
cd575ddf	21
61c8d7c8 KO	22	static inline unsigned stripe_csums_per_device(const struct bch_stripe *s)
	23	{
	24	return DIV_ROUND_UP(le16_to_cpu(s->sectors),
	25	1 << s->csum_granularity_bits);
	26	}
	27
	28	static inline unsigned stripe_csum_offset(const struct bch_stripe *s,
	29	unsigned dev, unsigned csum_idx)
	30	{
	31	unsigned csum_bytes = bch_crc_bytes[s->csum_type];
	32
	33	return sizeof(struct bch_stripe) +
	34	sizeof(struct bch_extent_ptr) * s->nr_blocks +
	35	(dev * stripe_csums_per_device(s) + csum_idx) * csum_bytes;
	36	}
	37
	38	static inline unsigned stripe_blockcount_offset(const struct bch_stripe *s,
	39	unsigned idx)
	40	{
	41	return stripe_csum_offset(s, s->nr_blocks, 0) +
	42	sizeof(u16) * idx;
	43	}
	44
	45	static inline unsigned stripe_blockcount_get(const struct bch_stripe *s,
	46	unsigned idx)
	47	{
	48	return le16_to_cpup((void *) s + stripe_blockcount_offset(s, idx));
	49	}
	50
	51	static inline void stripe_blockcount_set(struct bch_stripe *s,
	52	unsigned idx, unsigned v)
	53	{
	54	__le16 p = (void ) s + stripe_blockcount_offset(s, idx);
	55
	56	*p = cpu_to_le16(v);
	57	}
	58
	59	static inline unsigned stripe_val_u64s(const struct bch_stripe *s)
	60	{
	61	return DIV_ROUND_UP(stripe_blockcount_offset(s, s->nr_blocks),
	62	sizeof(u64));
	63	}
	64
	65	static inline void stripe_csum(struct bch_stripe s,
2a3731e3	66	unsigned block, unsigned csum_idx)
61c8d7c8	67	{
2a3731e3 KO	68	EBUG_ON(block >= s->nr_blocks);
	69	EBUG_ON(csum_idx >= stripe_csums_per_device(s));
	70
	71	return (void *) s + stripe_csum_offset(s, block, csum_idx);
	72	}
	73
	74	static inline struct bch_csum stripe_csum_get(struct bch_stripe *s,
	75	unsigned block, unsigned csum_idx)
	76	{
	77	struct bch_csum csum = { 0 };
	78
	79	memcpy(&csum, stripe_csum(s, block, csum_idx), bch_crc_bytes[s->csum_type]);
	80	return csum;
	81	}
	82
	83	static inline void stripe_csum_set(struct bch_stripe *s,
	84	unsigned block, unsigned csum_idx,
	85	struct bch_csum csum)
	86	{
	87	memcpy(stripe_csum(s, block, csum_idx), &csum, bch_crc_bytes[s->csum_type]);
	88	}
	89
b3b66e30 KO	90	static inline bool __bch2_ptr_matches_stripe(const struct bch_extent_ptr *stripe_ptr,
	91	const struct bch_extent_ptr *data_ptr,
	92	unsigned sectors)
	93	{
	94	return data_ptr->dev == stripe_ptr->dev &&
	95	data_ptr->gen == stripe_ptr->gen &&
	96	data_ptr->offset >= stripe_ptr->offset &&
	97	data_ptr->offset < stripe_ptr->offset + sectors;
	98	}
	99
	100	static inline bool bch2_ptr_matches_stripe(const struct bch_stripe *s,
	101	struct extent_ptr_decoded p)
2a3731e3 KO	102	{
	103	unsigned nr_data = s->nr_blocks - s->nr_redundant;
	104
b3b66e30 KO	105	BUG_ON(!p.has_ec);
	106
	107	if (p.ec.block >= nr_data)
2a3731e3 KO	108	return false;
2a3731e3 KO	109
b3b66e30 KO	110	return __bch2_ptr_matches_stripe(&s->ptrs[p.ec.block], &p.ptr,
b3b66e30 KO	111	le16_to_cpu(s->sectors));
2a3731e3 KO	112	}
2a3731e3 KO	113
990d42d1	114	static inline bool bch2_ptr_matches_stripe_m(const struct gc_stripe *m,
b3b66e30	115	struct extent_ptr_decoded p)
2a3731e3	116	{
b3b66e30 KO	117	unsigned nr_data = m->nr_blocks - m->nr_redundant;
b3b66e30 KO	118
2a3731e3 KO	119	BUG_ON(!p.has_ec);
2a3731e3 KO	120
b3b66e30 KO	121	if (p.ec.block >= nr_data)
	122	return false;
	123
	124	return __bch2_ptr_matches_stripe(&m->ptrs[p.ec.block], &p.ptr,
	125	m->sectors);
61c8d7c8 KO	126	}
61c8d7c8 KO	127
cd575ddf KO	128	struct bch_read_bio;
	129
	130	struct ec_stripe_buf {
	131	/* might not be buffering the entire stripe: */
	132	unsigned offset;
	133	unsigned size;
ffb7c3d3	134	unsigned long valid[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
cd575ddf	135
ffb7c3d3	136	void *data[BCH_BKEY_PTRS_MAX];
cd575ddf KO	137
	138	union {
	139	struct bkey_i_stripe key;
	140	u64 pad[255];
	141	};
	142	};
	143
	144	struct ec_stripe_head;
	145
	146	struct ec_stripe_new {
	147	struct bch_fs *c;
	148	struct ec_stripe_head *h;
	149	struct mutex lock;
	150	struct list_head list;
81d8599e	151	struct closure iodone;
cd575ddf KO	152
	153	/* counts in flight writes, stripe is created when pin == 0 */
	154	atomic_t pin;
	155
	156	int err;
	157
f6b94a3b KO	158	u8 nr_data;
	159	u8 nr_parity;
	160	bool allocated;
	161	bool pending;
81d8599e	162	bool have_existing_stripe;
0ba95acc	163
6c7585b0	164	unsigned long blocks_gotten[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
ffb7c3d3	165	unsigned long blocks_allocated[BITS_TO_LONGS(BCH_BKEY_PTRS_MAX)];
6c7585b0	166	open_bucket_idx_t blocks[BCH_BKEY_PTRS_MAX];
af4d05c4	167	struct disk_reservation res;
cd575ddf	168
81d8599e KO	169	struct ec_stripe_buf new_stripe;
81d8599e KO	170	struct ec_stripe_buf existing_stripe;
cd575ddf KO	171	};
	172
	173	struct ec_stripe_head {
	174	struct list_head list;
	175	struct mutex lock;
	176
cd575ddf KO	177	unsigned target;
	178	unsigned algo;
	179	unsigned redundancy;
8deed5f4	180	bool copygc;
cd575ddf KO	181
	182	struct bch_devs_mask devs;
	183	unsigned nr_active_devs;
	184
	185	unsigned blocksize;
	186
	187	struct dev_stripe_state block_stripe;
	188	struct dev_stripe_state parity_stripe;
	189
cd575ddf KO	190	struct ec_stripe_new *s;
	191	};
	192
	193	int bch2_ec_read_extent(struct bch_fs , struct bch_read_bio );
	194
	195	void bch2_writepoint_ec_buf(struct bch_fs , struct write_point *);
cd575ddf KO	196
	197	void bch2_ec_bucket_written(struct bch_fs , struct open_bucket );
	198	void bch2_ec_bucket_cancel(struct bch_fs , struct open_bucket );
	199
	200	int bch2_ec_stripe_new_alloc(struct bch_fs , struct ec_stripe_head );
	201
703e2a43	202	void bch2_ec_stripe_head_put(struct bch_fs , struct ec_stripe_head );
2c40a240	203	struct ec_stripe_head bch2_ec_stripe_head_get(struct bch_fs ,
8deed5f4	204	unsigned, unsigned, unsigned, bool, struct closure *);
cd575ddf	205
dfe9bfb3 KO	206	void bch2_stripes_heap_update(struct bch_fs , struct stripe , size_t);
	207	void bch2_stripes_heap_del(struct bch_fs , struct stripe , size_t);
	208	void bch2_stripes_heap_insert(struct bch_fs , struct stripe , size_t);
cd575ddf	209
dd81a060 KO	210	void bch2_do_stripe_deletes(struct bch_fs *);
dd81a060 KO	211
cd575ddf KO	212	void bch2_ec_stop_dev(struct bch_fs , struct bch_dev );
	213
	214	void bch2_ec_flush_new_stripes(struct bch_fs *);
	215
ac958006 KO	216	void bch2_stripes_heap_start(struct bch_fs *);
ac958006 KO	217
ac1019d3	218	int bch2_stripes_read(struct bch_fs *);
61c8d7c8	219
ba6dd1dd	220	void bch2_stripes_heap_to_text(struct printbuf , struct bch_fs );
7807e143	221	void bch2_new_stripes_to_text(struct printbuf , struct bch_fs );
ba6dd1dd	222
cd575ddf	223	void bch2_fs_ec_exit(struct bch_fs *);
84c72755	224	void bch2_fs_ec_init_early(struct bch_fs *);
cd575ddf KO	225	int bch2_fs_ec_init(struct bch_fs *);
	226
	227	#endif /* _BCACHEFS_EC_H */