[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / steering / dr_icm_pool.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2019 Mellanox Technologies. */

#include "dr_types.h"

#define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
#define DR_ICM_SYNC_THRESHOLD (64 * 1024 * 1024)

struct mlx5dr_icm_pool;

struct mlx5dr_icm_bucket {
	struct mlx5dr_icm_pool *pool;

	/* Chunks that aren't visible to HW not directly and not in cache */
	struct list_head free_list;
	unsigned int free_list_count;

	/* Used chunks, HW may be accessing this memory */
	struct list_head used_list;
	unsigned int used_list_count;

	/* HW may be accessing this memory but at some future,
	 * undetermined time, it might cease to do so. Before deciding to call
	 * sync_ste, this list is moved to sync_list
	 */
	struct list_head hot_list;
	unsigned int hot_list_count;

	/* Pending sync list, entries from the hot list are moved to this list.
	 * sync_ste is executed and then sync_list is concatenated to the free list
	 */
	struct list_head sync_list;
	unsigned int sync_list_count;

	u32 total_chunks;
	u32 num_of_entries;
	u32 entry_size;
	/* protect the ICM bucket */
	struct mutex mutex;
};

struct mlx5dr_icm_pool {
	struct mlx5dr_icm_bucket *buckets;
	enum mlx5dr_icm_type icm_type;
	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
	enum mlx5dr_icm_chunk_size num_of_buckets;
	struct list_head icm_mr_list;
	/* protect the ICM MR list */
	struct mutex mr_mutex;
	struct mlx5dr_domain *dmn;
};

struct mlx5dr_icm_dm {
	u32 obj_id;
	enum mlx5_sw_icm_type type;
	phys_addr_t addr;
	size_t length;
};

struct mlx5dr_icm_mr {
	struct mlx5dr_icm_pool *pool;
	struct mlx5_core_mkey mkey;
	struct mlx5dr_icm_dm dm;
	size_t used_length;
	size_t length;
	u64 icm_start_addr;
	struct list_head mr_list;
};

static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev,
				 u32 pd, u64 length, u64 start_addr, int mode,
				 struct mlx5_core_mkey *mkey)
{
	u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
	u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
	void *mkc;

	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

	MLX5_SET(mkc, mkc, access_mode_1_0, mode);
	MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
	MLX5_SET(mkc, mkc, lw, 1);
	MLX5_SET(mkc, mkc, lr, 1);
	if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) {
		MLX5_SET(mkc, mkc, rw, 1);
		MLX5_SET(mkc, mkc, rr, 1);
	}

	MLX5_SET64(mkc, mkc, len, length);
	MLX5_SET(mkc, mkc, pd, pd);
	MLX5_SET(mkc, mkc, qpn, 0xffffff);
	MLX5_SET64(mkc, mkc, start_addr, start_addr);

	return mlx5_core_create_mkey(mdev, mkey, in, inlen);
}

static struct mlx5dr_icm_mr *
dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool,
		      enum mlx5_sw_icm_type type,
		      size_t align_base)
{
	struct mlx5_core_dev *mdev = pool->dmn->mdev;
	struct mlx5dr_icm_mr *icm_mr;
	size_t align_diff;
	int err;

	icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL);
	if (!icm_mr)
		return NULL;

	icm_mr->pool = pool;
	INIT_LIST_HEAD(&icm_mr->mr_list);

	icm_mr->dm.type = type;

	/* 2^log_biggest_table * entry-size * double-for-alignment */
	icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
							       pool->icm_type) * 2;

	err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
				   &icm_mr->dm.addr, &icm_mr->dm.obj_id);
	if (err) {
		mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err);
		goto free_icm_mr;
	}

	/* Register device memory */
	err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn,
				    icm_mr->dm.length,
				    icm_mr->dm.addr,
				    MLX5_MKC_ACCESS_MODE_SW_ICM,
				    &icm_mr->mkey);
	if (err) {
		mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err);
		goto free_dm;
	}

	icm_mr->icm_start_addr = icm_mr->dm.addr;

	/* align_base is always a power of 2 */
	align_diff = icm_mr->icm_start_addr & (align_base - 1);
	if (align_diff)
		icm_mr->used_length = align_base - align_diff;

	list_add_tail(&icm_mr->mr_list, &pool->icm_mr_list);

	return icm_mr;

free_dm:
	mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
			       icm_mr->dm.addr, icm_mr->dm.obj_id);
free_icm_mr:
	kvfree(icm_mr);
	return NULL;
}

static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
{
	struct mlx5_core_dev *mdev = icm_mr->pool->dmn->mdev;
	struct mlx5dr_icm_dm *dm = &icm_mr->dm;

	list_del(&icm_mr->mr_list);
	mlx5_core_destroy_mkey(mdev, &icm_mr->mkey);
	mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0,
			       dm->addr, dm->obj_id);
	kvfree(icm_mr);
}

static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
{
	struct mlx5dr_icm_bucket *bucket = chunk->bucket;

	chunk->ste_arr = kvzalloc(bucket->num_of_entries *
				  sizeof(chunk->ste_arr[0]), GFP_KERNEL);
	if (!chunk->ste_arr)
		return -ENOMEM;

	chunk->hw_ste_arr = kvzalloc(bucket->num_of_entries *
				     DR_STE_SIZE_REDUCED, GFP_KERNEL);
	if (!chunk->hw_ste_arr)
		goto out_free_ste_arr;

	chunk->miss_list = kvmalloc(bucket->num_of_entries *
				    sizeof(chunk->miss_list[0]), GFP_KERNEL);
	if (!chunk->miss_list)
		goto out_free_hw_ste_arr;

	return 0;

out_free_hw_ste_arr:
	kvfree(chunk->hw_ste_arr);
out_free_ste_arr:
	kvfree(chunk->ste_arr);
	return -ENOMEM;
}

static int dr_icm_chunks_create(struct mlx5dr_icm_bucket *bucket)
{
	size_t mr_free_size, mr_req_size, mr_row_size;
	struct mlx5dr_icm_pool *pool = bucket->pool;
	struct mlx5dr_icm_mr *icm_mr = NULL;
	struct mlx5dr_icm_chunk *chunk;
	enum mlx5_sw_icm_type dm_type;
	size_t align_base;
	int i, err = 0;

	mr_req_size = bucket->num_of_entries * bucket->entry_size;
	mr_row_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
							 pool->icm_type);

	if (pool->icm_type == DR_ICM_TYPE_STE) {
		dm_type = MLX5_SW_ICM_TYPE_STEERING;
		/* Align base is the biggest chunk size / row size */
		align_base = mr_row_size;
	} else {
		dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY;
		/* Align base is 64B */
		align_base = DR_ICM_MODIFY_HDR_ALIGN_BASE;
	}

	mutex_lock(&pool->mr_mutex);
	if (!list_empty(&pool->icm_mr_list)) {
		icm_mr = list_last_entry(&pool->icm_mr_list,
					 struct mlx5dr_icm_mr, mr_list);

		if (icm_mr)
			mr_free_size = icm_mr->dm.length - icm_mr->used_length;
	}

	if (!icm_mr || mr_free_size < mr_row_size) {
		icm_mr = dr_icm_pool_mr_create(pool, dm_type, align_base);
		if (!icm_mr) {
			err = -ENOMEM;
			goto out_err;
		}
	}

	/* Create memory aligned chunks */
	for (i = 0; i < mr_row_size / mr_req_size; i++) {
		chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL);
		if (!chunk) {
			err = -ENOMEM;
			goto out_err;
		}

		chunk->bucket = bucket;
		chunk->rkey = icm_mr->mkey.key;
		/* mr start addr is zero based */
		chunk->mr_addr = icm_mr->used_length;
		chunk->icm_addr = (uintptr_t)icm_mr->icm_start_addr + icm_mr->used_length;
		icm_mr->used_length += mr_req_size;
		chunk->num_of_entries = bucket->num_of_entries;
		chunk->byte_size = chunk->num_of_entries * bucket->entry_size;

		if (pool->icm_type == DR_ICM_TYPE_STE) {
			err = dr_icm_chunk_ste_init(chunk);
			if (err)
				goto out_free_chunk;
		}

		INIT_LIST_HEAD(&chunk->chunk_list);
		list_add(&chunk->chunk_list, &bucket->free_list);
		bucket->free_list_count++;
		bucket->total_chunks++;
	}
	mutex_unlock(&pool->mr_mutex);
	return 0;

out_free_chunk:
	kvfree(chunk);
out_err:
	mutex_unlock(&pool->mr_mutex);
	return err;
}

static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
{
	kvfree(chunk->miss_list);
	kvfree(chunk->hw_ste_arr);
	kvfree(chunk->ste_arr);
}

static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk)
{
	struct mlx5dr_icm_bucket *bucket = chunk->bucket;

	list_del(&chunk->chunk_list);
	bucket->total_chunks--;

	if (bucket->pool->icm_type == DR_ICM_TYPE_STE)
		dr_icm_chunk_ste_cleanup(chunk);

	kvfree(chunk);
}

static void dr_icm_bucket_init(struct mlx5dr_icm_pool *pool,
			       struct mlx5dr_icm_bucket *bucket,
			       enum mlx5dr_icm_chunk_size chunk_size)
{
	if (pool->icm_type == DR_ICM_TYPE_STE)
		bucket->entry_size = DR_STE_SIZE;
	else
		bucket->entry_size = DR_MODIFY_ACTION_SIZE;

	bucket->num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
	bucket->pool = pool;
	mutex_init(&bucket->mutex);
	INIT_LIST_HEAD(&bucket->free_list);
	INIT_LIST_HEAD(&bucket->used_list);
	INIT_LIST_HEAD(&bucket->hot_list);
	INIT_LIST_HEAD(&bucket->sync_list);
}

static void dr_icm_bucket_cleanup(struct mlx5dr_icm_bucket *bucket)
{
	struct mlx5dr_icm_chunk *chunk, *next;

	mutex_destroy(&bucket->mutex);
	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
	list_splice_tail_init(&bucket->hot_list, &bucket->free_list);

	list_for_each_entry_safe(chunk, next, &bucket->free_list, chunk_list)
		dr_icm_chunk_destroy(chunk);

	WARN_ON(bucket->total_chunks != 0);

	/* Cleanup of unreturned chunks */
	list_for_each_entry_safe(chunk, next, &bucket->used_list, chunk_list)
		dr_icm_chunk_destroy(chunk);
}

static u64 dr_icm_hot_mem_size(struct mlx5dr_icm_pool *pool)
{
	u64 hot_size = 0;
	int chunk_order;

	for (chunk_order = 0; chunk_order < pool->num_of_buckets; chunk_order++)
		hot_size += pool->buckets[chunk_order].hot_list_count *
			    mlx5dr_icm_pool_chunk_size_to_byte(chunk_order, pool->icm_type);

	return hot_size;
}

static bool dr_icm_reuse_hot_entries(struct mlx5dr_icm_pool *pool,
				     struct mlx5dr_icm_bucket *bucket)
{
	u64 bytes_for_sync;

	bytes_for_sync = dr_icm_hot_mem_size(pool);
	if (bytes_for_sync < DR_ICM_SYNC_THRESHOLD || !bucket->hot_list_count)
		return false;

	return true;
}

static void dr_icm_chill_bucket_start(struct mlx5dr_icm_bucket *bucket)
{
	list_splice_tail_init(&bucket->hot_list, &bucket->sync_list);
	bucket->sync_list_count += bucket->hot_list_count;
	bucket->hot_list_count = 0;
}

static void dr_icm_chill_bucket_end(struct mlx5dr_icm_bucket *bucket)
{
	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
	bucket->free_list_count += bucket->sync_list_count;
	bucket->sync_list_count = 0;
}

static void dr_icm_chill_bucket_abort(struct mlx5dr_icm_bucket *bucket)
{
	list_splice_tail_init(&bucket->sync_list, &bucket->hot_list);
	bucket->hot_list_count += bucket->sync_list_count;
	bucket->sync_list_count = 0;
}

static void dr_icm_chill_buckets_start(struct mlx5dr_icm_pool *pool,
				       struct mlx5dr_icm_bucket *cb,
				       bool buckets[DR_CHUNK_SIZE_MAX])
{
	struct mlx5dr_icm_bucket *bucket;
	int i;

	for (i = 0; i < pool->num_of_buckets; i++) {
		bucket = &pool->buckets[i];
		if (bucket == cb) {
			dr_icm_chill_bucket_start(bucket);
			continue;
		}

		/* Freeing the mutex is done at the end of that process, after
		 * sync_ste was executed at dr_icm_chill_buckets_end func.
		 */
		if (mutex_trylock(&bucket->mutex)) {
			dr_icm_chill_bucket_start(bucket);
			buckets[i] = true;
		}
	}
}

static void dr_icm_chill_buckets_end(struct mlx5dr_icm_pool *pool,
				     struct mlx5dr_icm_bucket *cb,
				     bool buckets[DR_CHUNK_SIZE_MAX])
{
	struct mlx5dr_icm_bucket *bucket;
	int i;

	for (i = 0; i < pool->num_of_buckets; i++) {
		bucket = &pool->buckets[i];
		if (bucket == cb) {
			dr_icm_chill_bucket_end(bucket);
			continue;
		}

		if (!buckets[i])
			continue;

		dr_icm_chill_bucket_end(bucket);
		mutex_unlock(&bucket->mutex);
	}
}

static void dr_icm_chill_buckets_abort(struct mlx5dr_icm_pool *pool,
				       struct mlx5dr_icm_bucket *cb,
				       bool buckets[DR_CHUNK_SIZE_MAX])
{
	struct mlx5dr_icm_bucket *bucket;
	int i;

	for (i = 0; i < pool->num_of_buckets; i++) {
		bucket = &pool->buckets[i];
		if (bucket == cb) {
			dr_icm_chill_bucket_abort(bucket);
			continue;
		}

		if (!buckets[i])
			continue;

		dr_icm_chill_bucket_abort(bucket);
		mutex_unlock(&bucket->mutex);
	}
}

/* Allocate an ICM chunk, each chunk holds a piece of ICM memory and
 * also memory used for HW STE management for optimizations.
 */
struct mlx5dr_icm_chunk *
mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
		       enum mlx5dr_icm_chunk_size chunk_size)
{
	struct mlx5dr_icm_chunk *chunk = NULL; /* Fix compilation warning */
	bool buckets[DR_CHUNK_SIZE_MAX] = {};
	struct mlx5dr_icm_bucket *bucket;
	int err;

	if (chunk_size > pool->max_log_chunk_sz)
		return NULL;

	bucket = &pool->buckets[chunk_size];

	mutex_lock(&bucket->mutex);

	/* Take chunk from pool if available, otherwise allocate new chunks */
	if (list_empty(&bucket->free_list)) {
		if (dr_icm_reuse_hot_entries(pool, bucket)) {
			dr_icm_chill_buckets_start(pool, bucket, buckets);
			err = mlx5dr_cmd_sync_steering(pool->dmn->mdev);
			if (err) {
				dr_icm_chill_buckets_abort(pool, bucket, buckets);
				mlx5dr_dbg(pool->dmn, "Sync_steering failed\n");
				chunk = NULL;
				goto out;
			}
			dr_icm_chill_buckets_end(pool, bucket, buckets);
		} else {
			dr_icm_chunks_create(bucket);
		}
	}

	if (!list_empty(&bucket->free_list)) {
		chunk = list_last_entry(&bucket->free_list,
					struct mlx5dr_icm_chunk,
					chunk_list);
		if (chunk) {
			list_del_init(&chunk->chunk_list);
			list_add_tail(&chunk->chunk_list, &bucket->used_list);
			bucket->free_list_count--;
			bucket->used_list_count++;
		}
	}
out:
	mutex_unlock(&bucket->mutex);
	return chunk;
}

void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk)
{
	struct mlx5dr_icm_bucket *bucket = chunk->bucket;

	if (bucket->pool->icm_type == DR_ICM_TYPE_STE) {
		memset(chunk->ste_arr, 0,
		       bucket->num_of_entries * sizeof(chunk->ste_arr[0]));
		memset(chunk->hw_ste_arr, 0,
		       bucket->num_of_entries * DR_STE_SIZE_REDUCED);
	}

	mutex_lock(&bucket->mutex);
	list_del_init(&chunk->chunk_list);
	list_add_tail(&chunk->chunk_list, &bucket->hot_list);
	bucket->hot_list_count++;
	bucket->used_list_count--;
	mutex_unlock(&bucket->mutex);
}

struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
					       enum mlx5dr_icm_type icm_type)
{
	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
	struct mlx5dr_icm_pool *pool;
	int i;

	if (icm_type == DR_ICM_TYPE_STE)
		max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
	else
		max_log_chunk_sz = dmn->info.max_log_action_icm_sz;

	pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
	if (!pool)
		return NULL;

	pool->buckets = kcalloc(max_log_chunk_sz + 1,
				sizeof(pool->buckets[0]),
				GFP_KERNEL);
	if (!pool->buckets)
		goto free_pool;

	pool->dmn = dmn;
	pool->icm_type = icm_type;
	pool->max_log_chunk_sz = max_log_chunk_sz;
	pool->num_of_buckets = max_log_chunk_sz + 1;
	INIT_LIST_HEAD(&pool->icm_mr_list);

	for (i = 0; i < pool->num_of_buckets; i++)
		dr_icm_bucket_init(pool, &pool->buckets[i], i);

	mutex_init(&pool->mr_mutex);

	return pool;

free_pool:
	kvfree(pool);
	return NULL;
}

void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool)
{
	struct mlx5dr_icm_mr *icm_mr, *next;
	int i;

	mutex_destroy(&pool->mr_mutex);

	list_for_each_entry_safe(icm_mr, next, &pool->icm_mr_list, mr_list)
		dr_icm_pool_mr_destroy(icm_mr);

	for (i = 0; i < pool->num_of_buckets; i++)
		dr_icm_bucket_cleanup(&pool->buckets[i]);

	kfree(pool->buckets);
	kvfree(pool);
}
Commit	Line	Data
29cf8feb AV	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2019 Mellanox Technologies. */
	3
	4	#include "dr_types.h"
	5
	6	#define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
	7	#define DR_ICM_SYNC_THRESHOLD (64 * 1024 * 1024)
	8
	9	struct mlx5dr_icm_pool;
	10
	11	struct mlx5dr_icm_bucket {
	12	struct mlx5dr_icm_pool *pool;
	13
	14	/* Chunks that aren't visible to HW not directly and not in cache */
	15	struct list_head free_list;
	16	unsigned int free_list_count;
	17
	18	/* Used chunks, HW may be accessing this memory */
	19	struct list_head used_list;
	20	unsigned int used_list_count;
	21
	22	/* HW may be accessing this memory but at some future,
	23	* undetermined time, it might cease to do so. Before deciding to call
	24	* sync_ste, this list is moved to sync_list
	25	*/
	26	struct list_head hot_list;
	27	unsigned int hot_list_count;
	28
	29	/* Pending sync list, entries from the hot list are moved to this list.
	30	* sync_ste is executed and then sync_list is concatenated to the free list
	31	*/
	32	struct list_head sync_list;
	33	unsigned int sync_list_count;
	34
	35	u32 total_chunks;
	36	u32 num_of_entries;
	37	u32 entry_size;
	38	/* protect the ICM bucket */
	39	struct mutex mutex;
	40	};
	41
	42	struct mlx5dr_icm_pool {
	43	struct mlx5dr_icm_bucket *buckets;
	44	enum mlx5dr_icm_type icm_type;
	45	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
	46	enum mlx5dr_icm_chunk_size num_of_buckets;
	47	struct list_head icm_mr_list;
	48	/* protect the ICM MR list */
	49	struct mutex mr_mutex;
	50	struct mlx5dr_domain *dmn;
	51	};
	52
	53	struct mlx5dr_icm_dm {
	54	u32 obj_id;
	55	enum mlx5_sw_icm_type type;
334a306f	56	phys_addr_t addr;
29cf8feb AV	57	size_t length;
	58	};
	59
	60	struct mlx5dr_icm_mr {
	61	struct mlx5dr_icm_pool *pool;
	62	struct mlx5_core_mkey mkey;
	63	struct mlx5dr_icm_dm dm;
	64	size_t used_length;
	65	size_t length;
	66	u64 icm_start_addr;
	67	struct list_head mr_list;
	68	};
	69
	70	static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev,
	71	u32 pd, u64 length, u64 start_addr, int mode,
	72	struct mlx5_core_mkey *mkey)
	73	{
	74	u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
	75	u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
	76	void *mkc;
	77
	78	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
	79
	80	MLX5_SET(mkc, mkc, access_mode_1_0, mode);
	81	MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
	82	MLX5_SET(mkc, mkc, lw, 1);
	83	MLX5_SET(mkc, mkc, lr, 1);
	84	if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) {
	85	MLX5_SET(mkc, mkc, rw, 1);
	86	MLX5_SET(mkc, mkc, rr, 1);
	87	}
	88
	89	MLX5_SET64(mkc, mkc, len, length);
	90	MLX5_SET(mkc, mkc, pd, pd);
	91	MLX5_SET(mkc, mkc, qpn, 0xffffff);
	92	MLX5_SET64(mkc, mkc, start_addr, start_addr);
	93
	94	return mlx5_core_create_mkey(mdev, mkey, in, inlen);
	95	}
	96
	97	static struct mlx5dr_icm_mr *
	98	dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool,
	99	enum mlx5_sw_icm_type type,
	100	size_t align_base)
	101	{
	102	struct mlx5_core_dev *mdev = pool->dmn->mdev;
	103	struct mlx5dr_icm_mr *icm_mr;
	104	size_t align_diff;
	105	int err;
	106
	107	icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL);
	108	if (!icm_mr)
	109	return NULL;
	110
	111	icm_mr->pool = pool;
	112	INIT_LIST_HEAD(&icm_mr->mr_list);
	113
	114	icm_mr->dm.type = type;
	115
	116	/* 2^log_biggest_table * entry-size * double-for-alignment */
	117	icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
	118	pool->icm_type) * 2;
	119
	120	err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
121	&icm_mr->dm.addr, &icm_mr->dm.obj_id);
122	if (err) {
123	mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err);
124	goto free_icm_mr;
125	}
126
127	/* Register device memory */
128	err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn,
129	icm_mr->dm.length,
130	icm_mr->dm.addr,
131	MLX5_MKC_ACCESS_MODE_SW_ICM,
132	&icm_mr->mkey);
133	if (err) {
134	mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err);
135	goto free_dm;
136	}
137
138	icm_mr->icm_start_addr = icm_mr->dm.addr;
139
8b6b82ad MK	140	/* align_base is always a power of 2 */
8b6b82ad MK	141	align_diff = icm_mr->icm_start_addr & (align_base - 1);
29cf8feb AV	142	if (align_diff)
	143	icm_mr->used_length = align_base - align_diff;
	144
	145	list_add_tail(&icm_mr->mr_list, &pool->icm_mr_list);
	146
	147	return icm_mr;
	148
	149	free_dm:
	150	mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
	151	icm_mr->dm.addr, icm_mr->dm.obj_id);
	152	free_icm_mr:
	153	kvfree(icm_mr);
	154	return NULL;
	155	}
	156
	157	static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
	158	{
	159	struct mlx5_core_dev *mdev = icm_mr->pool->dmn->mdev;
	160	struct mlx5dr_icm_dm *dm = &icm_mr->dm;
	161
	162	list_del(&icm_mr->mr_list);
	163	mlx5_core_destroy_mkey(mdev, &icm_mr->mkey);
	164	mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0,
	165	dm->addr, dm->obj_id);
	166	kvfree(icm_mr);
	167	}
	168
	169	static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
	170	{
	171	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
	172
	173	chunk->ste_arr = kvzalloc(bucket->num_of_entries *
	174	sizeof(chunk->ste_arr[0]), GFP_KERNEL);
	175	if (!chunk->ste_arr)
	176	return -ENOMEM;
	177
	178	chunk->hw_ste_arr = kvzalloc(bucket->num_of_entries *
	179	DR_STE_SIZE_REDUCED, GFP_KERNEL);
	180	if (!chunk->hw_ste_arr)
	181	goto out_free_ste_arr;
	182
	183	chunk->miss_list = kvmalloc(bucket->num_of_entries *
	184	sizeof(chunk->miss_list[0]), GFP_KERNEL);
	185	if (!chunk->miss_list)
	186	goto out_free_hw_ste_arr;
	187
	188	return 0;
	189
	190	out_free_hw_ste_arr:
	191	kvfree(chunk->hw_ste_arr);
	192	out_free_ste_arr:
	193	kvfree(chunk->ste_arr);
	194	return -ENOMEM;
	195	}
	196
	197	static int dr_icm_chunks_create(struct mlx5dr_icm_bucket *bucket)
	198	{
	199	size_t mr_free_size, mr_req_size, mr_row_size;
	200	struct mlx5dr_icm_pool *pool = bucket->pool;
	201	struct mlx5dr_icm_mr *icm_mr = NULL;
	202	struct mlx5dr_icm_chunk *chunk;
	203	enum mlx5_sw_icm_type dm_type;
	204	size_t align_base;
	205	int i, err = 0;
206
207	mr_req_size = bucket->num_of_entries * bucket->entry_size;
208	mr_row_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
209	pool->icm_type);
210
211	if (pool->icm_type == DR_ICM_TYPE_STE) {
212	dm_type = MLX5_SW_ICM_TYPE_STEERING;
213	/* Align base is the biggest chunk size / row size */
214	align_base = mr_row_size;
215	} else {
216	dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY;
217	/* Align base is 64B */
218	align_base = DR_ICM_MODIFY_HDR_ALIGN_BASE;
219	}
220
221	mutex_lock(&pool->mr_mutex);
222	if (!list_empty(&pool->icm_mr_list)) {
223	icm_mr = list_last_entry(&pool->icm_mr_list,
224	struct mlx5dr_icm_mr, mr_list);
225
226	if (icm_mr)
227	mr_free_size = icm_mr->dm.length - icm_mr->used_length;
228	}
229
230	if (!icm_mr \|\| mr_free_size < mr_row_size) {
231	icm_mr = dr_icm_pool_mr_create(pool, dm_type, align_base);
232	if (!icm_mr) {
233	err = -ENOMEM;
234	goto out_err;
235	}
236	}
237
238	/* Create memory aligned chunks */
239	for (i = 0; i < mr_row_size / mr_req_size; i++) {
240	chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL);
241	if (!chunk) {
242	err = -ENOMEM;
243	goto out_err;
244	}
245
246	chunk->bucket = bucket;
247	chunk->rkey = icm_mr->mkey.key;
248	/* mr start addr is zero based */
249	chunk->mr_addr = icm_mr->used_length;
250	chunk->icm_addr = (uintptr_t)icm_mr->icm_start_addr + icm_mr->used_length;
251	icm_mr->used_length += mr_req_size;
252	chunk->num_of_entries = bucket->num_of_entries;
253	chunk->byte_size = chunk->num_of_entries * bucket->entry_size;
254
255	if (pool->icm_type == DR_ICM_TYPE_STE) {
256	err = dr_icm_chunk_ste_init(chunk);
257	if (err)
258	goto out_free_chunk;
259	}
260
261	INIT_LIST_HEAD(&chunk->chunk_list);
262	list_add(&chunk->chunk_list, &bucket->free_list);
263	bucket->free_list_count++;
264	bucket->total_chunks++;
265	}
266	mutex_unlock(&pool->mr_mutex);
267	return 0;
268
269	out_free_chunk:
270	kvfree(chunk);
271	out_err:
272	mutex_unlock(&pool->mr_mutex);
273	return err;
274	}
275
276	static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
277	{
278	kvfree(chunk->miss_list);
279	kvfree(chunk->hw_ste_arr);
280	kvfree(chunk->ste_arr);
281	}
282
283	static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk)
284	{
285	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
286
287	list_del(&chunk->chunk_list);
288	bucket->total_chunks--;
289
290	if (bucket->pool->icm_type == DR_ICM_TYPE_STE)
291	dr_icm_chunk_ste_cleanup(chunk);
292
293	kvfree(chunk);
294	}
295
296	static void dr_icm_bucket_init(struct mlx5dr_icm_pool *pool,
297	struct mlx5dr_icm_bucket *bucket,
298	enum mlx5dr_icm_chunk_size chunk_size)
299	{
300	if (pool->icm_type == DR_ICM_TYPE_STE)
301	bucket->entry_size = DR_STE_SIZE;
302	else
303	bucket->entry_size = DR_MODIFY_ACTION_SIZE;
304
305	bucket->num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
306	bucket->pool = pool;
307	mutex_init(&bucket->mutex);
308	INIT_LIST_HEAD(&bucket->free_list);
309	INIT_LIST_HEAD(&bucket->used_list);
310	INIT_LIST_HEAD(&bucket->hot_list);
311	INIT_LIST_HEAD(&bucket->sync_list);
312	}
313
314	static void dr_icm_bucket_cleanup(struct mlx5dr_icm_bucket *bucket)
315	{
316	struct mlx5dr_icm_chunk chunk, next;
317
318	mutex_destroy(&bucket->mutex);
319	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
320	list_splice_tail_init(&bucket->hot_list, &bucket->free_list);
321
322	list_for_each_entry_safe(chunk, next, &bucket->free_list, chunk_list)
323	dr_icm_chunk_destroy(chunk);
324
325	WARN_ON(bucket->total_chunks != 0);
326
327	/* Cleanup of unreturned chunks */
328	list_for_each_entry_safe(chunk, next, &bucket->used_list, chunk_list)
329	dr_icm_chunk_destroy(chunk);
330	}
331
332	static u64 dr_icm_hot_mem_size(struct mlx5dr_icm_pool *pool)
333	{
334	u64 hot_size = 0;
335	int chunk_order;
336
337	for (chunk_order = 0; chunk_order < pool->num_of_buckets; chunk_order++)
338	hot_size += pool->buckets[chunk_order].hot_list_count *
339	mlx5dr_icm_pool_chunk_size_to_byte(chunk_order, pool->icm_type);
340
341	return hot_size;
342	}
343
344	static bool dr_icm_reuse_hot_entries(struct mlx5dr_icm_pool *pool,
345	struct mlx5dr_icm_bucket *bucket)
346	{
347	u64 bytes_for_sync;
348
349	bytes_for_sync = dr_icm_hot_mem_size(pool);
350	if (bytes_for_sync < DR_ICM_SYNC_THRESHOLD \|\| !bucket->hot_list_count)
351	return false;
352
353	return true;
354	}
355
356	static void dr_icm_chill_bucket_start(struct mlx5dr_icm_bucket *bucket)
357	{
358	list_splice_tail_init(&bucket->hot_list, &bucket->sync_list);
359	bucket->sync_list_count += bucket->hot_list_count;
360	bucket->hot_list_count = 0;
361	}
362
363	static void dr_icm_chill_bucket_end(struct mlx5dr_icm_bucket *bucket)
364	{
365	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
366	bucket->free_list_count += bucket->sync_list_count;
367	bucket->sync_list_count = 0;
368	}
369
370	static void dr_icm_chill_bucket_abort(struct mlx5dr_icm_bucket *bucket)
371	{
372	list_splice_tail_init(&bucket->sync_list, &bucket->hot_list);
373	bucket->hot_list_count += bucket->sync_list_count;
374	bucket->sync_list_count = 0;
375	}
376
377	static void dr_icm_chill_buckets_start(struct mlx5dr_icm_pool *pool,
378	struct mlx5dr_icm_bucket *cb,
379	bool buckets[DR_CHUNK_SIZE_MAX])
380	{
381	struct mlx5dr_icm_bucket *bucket;
382	int i;
383
384	for (i = 0; i < pool->num_of_buckets; i++) {
385	bucket = &pool->buckets[i];
386	if (bucket == cb) {
387	dr_icm_chill_bucket_start(bucket);
388	continue;
389	}
390
391	/* Freeing the mutex is done at the end of that process, after
392	* sync_ste was executed at dr_icm_chill_buckets_end func.
393	*/
394	if (mutex_trylock(&bucket->mutex)) {
395	dr_icm_chill_bucket_start(bucket);
396	buckets[i] = true;
397	}
398	}
399	}
400
401	static void dr_icm_chill_buckets_end(struct mlx5dr_icm_pool *pool,
402	struct mlx5dr_icm_bucket *cb,
403	bool buckets[DR_CHUNK_SIZE_MAX])
404	{
405	struct mlx5dr_icm_bucket *bucket;
406	int i;
407
408	for (i = 0; i < pool->num_of_buckets; i++) {
409	bucket = &pool->buckets[i];
410	if (bucket == cb) {
411	dr_icm_chill_bucket_end(bucket);
412	continue;
413	}
414
415	if (!buckets[i])
416	continue;
417
418	dr_icm_chill_bucket_end(bucket);
419	mutex_unlock(&bucket->mutex);
420	}
421	}
422
423	static void dr_icm_chill_buckets_abort(struct mlx5dr_icm_pool *pool,
424	struct mlx5dr_icm_bucket *cb,
425	bool buckets[DR_CHUNK_SIZE_MAX])
426	{
427	struct mlx5dr_icm_bucket *bucket;
428	int i;
429
430	for (i = 0; i < pool->num_of_buckets; i++) {
431	bucket = &pool->buckets[i];
432	if (bucket == cb) {
433	dr_icm_chill_bucket_abort(bucket);
434	continue;
435	}
436
437	if (!buckets[i])
438	continue;
439
440	dr_icm_chill_bucket_abort(bucket);
441	mutex_unlock(&bucket->mutex);
442	}
443	}
444
445	/* Allocate an ICM chunk, each chunk holds a piece of ICM memory and
446	* also memory used for HW STE management for optimizations.
447	*/
448	struct mlx5dr_icm_chunk *
449	mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
450	enum mlx5dr_icm_chunk_size chunk_size)
451	{
452	struct mlx5dr_icm_chunk chunk = NULL; / Fix compilation warning */
453	bool buckets[DR_CHUNK_SIZE_MAX] = {};
454	struct mlx5dr_icm_bucket *bucket;
455	int err;
456
457	if (chunk_size > pool->max_log_chunk_sz)
458	return NULL;
459
460	bucket = &pool->buckets[chunk_size];
461
462	mutex_lock(&bucket->mutex);
463
464	/* Take chunk from pool if available, otherwise allocate new chunks */
465	if (list_empty(&bucket->free_list)) {
466	if (dr_icm_reuse_hot_entries(pool, bucket)) {
467	dr_icm_chill_buckets_start(pool, bucket, buckets);
468	err = mlx5dr_cmd_sync_steering(pool->dmn->mdev);
469	if (err) {
470	dr_icm_chill_buckets_abort(pool, bucket, buckets);
471	mlx5dr_dbg(pool->dmn, "Sync_steering failed\n");
472	chunk = NULL;
473	goto out;
474	}
475	dr_icm_chill_buckets_end(pool, bucket, buckets);
476	} else {
477	dr_icm_chunks_create(bucket);
478	}
479	}
480
481	if (!list_empty(&bucket->free_list)) {
482	chunk = list_last_entry(&bucket->free_list,
483	struct mlx5dr_icm_chunk,
484	chunk_list);
485	if (chunk) {
486	list_del_init(&chunk->chunk_list);
487	list_add_tail(&chunk->chunk_list, &bucket->used_list);
488	bucket->free_list_count--;
489	bucket->used_list_count++;
490	}
491	}
492	out:
493	mutex_unlock(&bucket->mutex);
494	return chunk;
495	}
496
497	void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk)
498	{
499	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
500
501	if (bucket->pool->icm_type == DR_ICM_TYPE_STE) {
502	memset(chunk->ste_arr, 0,
503	bucket->num_of_entries * sizeof(chunk->ste_arr[0]));
504	memset(chunk->hw_ste_arr, 0,
505	bucket->num_of_entries * DR_STE_SIZE_REDUCED);
506	}
507
508	mutex_lock(&bucket->mutex);
509	list_del_init(&chunk->chunk_list);
510	list_add_tail(&chunk->chunk_list, &bucket->hot_list);
511	bucket->hot_list_count++;
512	bucket->used_list_count--;
513	mutex_unlock(&bucket->mutex);
514	}
515
516	struct mlx5dr_icm_pool mlx5dr_icm_pool_create(struct mlx5dr_domain dmn,
517	enum mlx5dr_icm_type icm_type)
518	{
519	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
520	struct mlx5dr_icm_pool *pool;
521	int i;
522
523	if (icm_type == DR_ICM_TYPE_STE)
524	max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
525	else
526	max_log_chunk_sz = dmn->info.max_log_action_icm_sz;
527
528	pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
529	if (!pool)
530	return NULL;
531
532	pool->buckets = kcalloc(max_log_chunk_sz + 1,
533	sizeof(pool->buckets[0]),
534	GFP_KERNEL);
535	if (!pool->buckets)
536	goto free_pool;
537
538	pool->dmn = dmn;
539	pool->icm_type = icm_type;
540	pool->max_log_chunk_sz = max_log_chunk_sz;
541	pool->num_of_buckets = max_log_chunk_sz + 1;
542	INIT_LIST_HEAD(&pool->icm_mr_list);
543
544	for (i = 0; i < pool->num_of_buckets; i++)
545	dr_icm_bucket_init(pool, &pool->buckets[i], i);
546
547	mutex_init(&pool->mr_mutex);
548
549	return pool;
550
551	free_pool:
552	kvfree(pool);
553	return NULL;
554	}
555
556	void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool)
557	{
558	struct mlx5dr_icm_mr icm_mr, next;
559	int i;
560
561	mutex_destroy(&pool->mr_mutex);
562
563	list_for_each_entry_safe(icm_mr, next, &pool->icm_mr_list, mr_list)
564	dr_icm_pool_mr_destroy(icm_mr);
565
566	for (i = 0; i < pool->num_of_buckets; i++)
567	dr_icm_bucket_cleanup(&pool->buckets[i]);
568
569	kfree(pool->buckets);
570	kvfree(pool);
571	}