Commit | Line | Data |
---|---|---|
e088a685 YL |
1 | /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ |
2 | /* | |
3 | * Copyright (c) 2017 Hisilicon Limited. | |
4 | * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. | |
5 | */ | |
6 | ||
e088a685 YL |
7 | #include <rdma/ib_umem.h> |
8 | #include "hns_roce_device.h" | |
9 | ||
69e0a42f | 10 | int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, |
e088a685 YL |
11 | struct hns_roce_db *db) |
12 | { | |
60c3becf | 13 | unsigned long page_addr = virt & PAGE_MASK; |
e088a685 | 14 | struct hns_roce_user_db_page *page; |
60c3becf | 15 | unsigned int offset; |
e088a685 YL |
16 | int ret = 0; |
17 | ||
18 | mutex_lock(&context->page_mutex); | |
19 | ||
20 | list_for_each_entry(page, &context->page_list, list) | |
60c3becf | 21 | if (page->user_virt == page_addr) |
e088a685 YL |
22 | goto found; |
23 | ||
24 | page = kmalloc(sizeof(*page), GFP_KERNEL); | |
25 | if (!page) { | |
26 | ret = -ENOMEM; | |
27 | goto out; | |
28 | } | |
29 | ||
30 | refcount_set(&page->refcount, 1); | |
60c3becf | 31 | page->user_virt = page_addr; |
c320e527 MS |
32 | page->umem = ib_umem_get(context->ibucontext.device, page_addr, |
33 | PAGE_SIZE, 0); | |
e088a685 YL |
34 | if (IS_ERR(page->umem)) { |
35 | ret = PTR_ERR(page->umem); | |
36 | kfree(page); | |
37 | goto out; | |
38 | } | |
39 | ||
40 | list_add(&page->list, &context->page_list); | |
41 | ||
42 | found: | |
60c3becf | 43 | offset = virt - page_addr; |
79fbd3e1 MG |
44 | db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset; |
45 | db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset; | |
e088a685 YL |
46 | db->u.user_page = page; |
47 | refcount_inc(&page->refcount); | |
48 | ||
49 | out: | |
50 | mutex_unlock(&context->page_mutex); | |
51 | ||
52 | return ret; | |
53 | } | |
e088a685 YL |
54 | |
55 | void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, | |
56 | struct hns_roce_db *db) | |
57 | { | |
58 | mutex_lock(&context->page_mutex); | |
59 | ||
60 | refcount_dec(&db->u.user_page->refcount); | |
61 | if (refcount_dec_if_one(&db->u.user_page->refcount)) { | |
62 | list_del(&db->u.user_page->list); | |
63 | ib_umem_release(db->u.user_page->umem); | |
64 | kfree(db->u.user_page); | |
65 | } | |
66 | ||
67 | mutex_unlock(&context->page_mutex); | |
68 | } | |
472bc0fb YL |
69 | |
70 | static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( | |
71 | struct device *dma_device) | |
72 | { | |
73 | struct hns_roce_db_pgdir *pgdir; | |
74 | ||
75 | pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); | |
76 | if (!pgdir) | |
77 | return NULL; | |
78 | ||
2a3d923f LO |
79 | bitmap_fill(pgdir->order1, |
80 | HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); | |
472bc0fb YL |
81 | pgdir->bits[0] = pgdir->order0; |
82 | pgdir->bits[1] = pgdir->order1; | |
83 | pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, | |
84 | &pgdir->db_dma, GFP_KERNEL); | |
85 | if (!pgdir->page) { | |
86 | kfree(pgdir); | |
87 | return NULL; | |
88 | } | |
89 | ||
90 | return pgdir; | |
91 | } | |
92 | ||
93 | static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir, | |
94 | struct hns_roce_db *db, int order) | |
95 | { | |
dcdc366a WL |
96 | unsigned long o; |
97 | unsigned long i; | |
472bc0fb YL |
98 | |
99 | for (o = order; o <= 1; ++o) { | |
100 | i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o); | |
101 | if (i < HNS_ROCE_DB_PER_PAGE >> o) | |
102 | goto found; | |
103 | } | |
104 | ||
105 | return -ENOMEM; | |
106 | ||
107 | found: | |
108 | clear_bit(i, pgdir->bits[o]); | |
109 | ||
110 | i <<= o; | |
111 | ||
112 | if (o > order) | |
113 | set_bit(i ^ 1, pgdir->bits[order]); | |
114 | ||
115 | db->u.pgdir = pgdir; | |
116 | db->index = i; | |
117 | db->db_record = pgdir->page + db->index; | |
2a3d923f | 118 | db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; |
472bc0fb YL |
119 | db->order = order; |
120 | ||
121 | return 0; | |
122 | } | |
123 | ||
124 | int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db, | |
125 | int order) | |
126 | { | |
127 | struct hns_roce_db_pgdir *pgdir; | |
128 | int ret = 0; | |
129 | ||
130 | mutex_lock(&hr_dev->pgdir_mutex); | |
131 | ||
132 | list_for_each_entry(pgdir, &hr_dev->pgdir_list, list) | |
133 | if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order)) | |
134 | goto out; | |
135 | ||
136 | pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev); | |
137 | if (!pgdir) { | |
138 | ret = -ENOMEM; | |
139 | goto out; | |
140 | } | |
141 | ||
142 | list_add(&pgdir->list, &hr_dev->pgdir_list); | |
143 | ||
144 | /* This should never fail -- we just allocated an empty page: */ | |
145 | WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order)); | |
146 | ||
147 | out: | |
148 | mutex_unlock(&hr_dev->pgdir_mutex); | |
149 | ||
150 | return ret; | |
151 | } | |
472bc0fb YL |
152 | |
153 | void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) | |
154 | { | |
dcdc366a WL |
155 | unsigned long o; |
156 | unsigned long i; | |
472bc0fb YL |
157 | |
158 | mutex_lock(&hr_dev->pgdir_mutex); | |
159 | ||
160 | o = db->order; | |
161 | i = db->index; | |
162 | ||
163 | if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { | |
164 | clear_bit(i ^ 1, db->u.pgdir->order0); | |
165 | ++o; | |
166 | } | |
167 | ||
168 | i >>= o; | |
169 | set_bit(i, db->u.pgdir->bits[o]); | |
170 | ||
2a3d923f LO |
171 | if (bitmap_full(db->u.pgdir->order1, |
172 | HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { | |
472bc0fb YL |
173 | dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, |
174 | db->u.pgdir->db_dma); | |
175 | list_del(&db->u.pgdir->list); | |
176 | kfree(db->u.pgdir); | |
177 | } | |
178 | ||
179 | mutex_unlock(&hr_dev->pgdir_mutex); | |
180 | } |