da15bb4511788d643334a2805eb05692296ecf4a
[linux-2.6-block.git] / drivers / net / ethernet / marvell / octeontx2 / af / rvu_npc.c
1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Admin Function driver
3  *
4  * Copyright (C) 2018 Marvell.
5  *
6  */
7
8 #include <linux/bitfield.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11
12 #include "rvu_struct.h"
13 #include "rvu_reg.h"
14 #include "rvu.h"
15 #include "npc.h"
16 #include "cgx.h"
17 #include "npc_profile.h"
18 #include "rvu_npc_hash.h"
19
20 #define RSVD_MCAM_ENTRIES_PER_PF        3 /* Broadcast, Promisc and AllMulticast */
21 #define RSVD_MCAM_ENTRIES_PER_NIXLF     1 /* Ucast for LFs */
22
23 #define NPC_PARSE_RESULT_DMAC_OFFSET    8
24 #define NPC_HW_TSTAMP_OFFSET            8ULL
25 #define NPC_KEX_CHAN_MASK               0xFFFULL
26 #define NPC_KEX_PF_FUNC_MASK            0xFFFFULL
27
28 #define ALIGN_8B_CEIL(__a)      (((__a) + 7) & (-8))
29
30 static const char def_pfl_name[] = "default";
31
32 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,
33                                       int blkaddr, u16 pcifunc);
34 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam,
35                                        u16 pcifunc);
36
37 bool is_npc_intf_tx(u8 intf)
38 {
39         return !!(intf & 0x1);
40 }
41
42 bool is_npc_intf_rx(u8 intf)
43 {
44         return !(intf & 0x1);
45 }
46
47 bool is_npc_interface_valid(struct rvu *rvu, u8 intf)
48 {
49         struct rvu_hwinfo *hw = rvu->hw;
50
51         return intf < hw->npc_intfs;
52 }
53
54 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena)
55 {
56         /* Due to a HW issue in these silicon versions, parse nibble enable
57          * configuration has to be identical for both Rx and Tx interfaces.
58          */
59         if (is_rvu_96xx_B0(rvu))
60                 return nibble_ena;
61         return 0;
62 }
63
64 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf)
65 {
66         int blkaddr;
67         u64 val = 0;
68
69         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
70         if (blkaddr < 0)
71                 return;
72
73         /* Config CPI base for the PKIND */
74         val = pkind | 1ULL << 62;
75         rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val);
76 }
77
78 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf)
79 {
80         struct npc_pkind *pkind = &rvu->hw->pkind;
81         u32 map;
82         int i;
83
84         for (i = 0; i < pkind->rsrc.max; i++) {
85                 map = pkind->pfchan_map[i];
86                 if (((map >> 16) & 0x3F) == pf)
87                         return i;
88         }
89         return -1;
90 }
91
92 #define NPC_AF_ACTION0_PTR_ADVANCE      GENMASK_ULL(27, 20)
93
94 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable)
95 {
96         int pkind, blkaddr;
97         u64 val;
98
99         pkind = rvu_npc_get_pkind(rvu, pf);
100         if (pkind < 0) {
101                 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__);
102                 return -EINVAL;
103         }
104
105         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);
106         if (blkaddr < 0) {
107                 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
108                 return -EINVAL;
109         }
110
111         val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));
112         val &= ~NPC_AF_ACTION0_PTR_ADVANCE;
113         /* If timestamp is enabled then configure NPC to shift 8 bytes */
114         if (enable)
115                 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE,
116                                   NPC_HW_TSTAMP_OFFSET);
117         rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);
118
119         return 0;
120 }
121
122 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc,
123                                     int nixlf)
124 {
125         struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam);
126         struct rvu *rvu = hw->rvu;
127         int blkaddr = 0, max = 0;
128         struct rvu_block *block;
129         struct rvu_pfvf *pfvf;
130
131         pfvf = rvu_get_pfvf(rvu, pcifunc);
132         /* Given a PF/VF and NIX LF number calculate the unicast mcam
133          * entry index based on the NIX block assigned to the PF/VF.
134          */
135         blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
136         while (blkaddr) {
137                 if (pfvf->nix_blkaddr == blkaddr)
138                         break;
139                 block = &rvu->hw->block[blkaddr];
140                 max += block->lf.max;
141                 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);
142         }
143
144         return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF;
145 }
146
147 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,
148                              u16 pcifunc, int nixlf, int type)
149 {
150         int pf = rvu_get_pf(pcifunc);
151         int index;
152
153         /* Check if this is for a PF */
154         if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) {
155                 /* Reserved entries exclude PF0 */
156                 pf--;
157                 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF);
158                 /* Broadcast address matching entry should be first so
159                  * that the packet can be replicated to all VFs.
160                  */
161                 if (type == NIXLF_BCAST_ENTRY)
162                         return index;
163                 else if (type == NIXLF_ALLMULTI_ENTRY)
164                         return index + 1;
165                 else if (type == NIXLF_PROMISC_ENTRY)
166                         return index + 2;
167         }
168
169         return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf);
170 }
171
172 int npc_get_bank(struct npc_mcam *mcam, int index)
173 {
174         int bank = index / mcam->banksize;
175
176         /* 0,1 & 2,3 banks are combined for this keysize */
177         if (mcam->keysize == NPC_MCAM_KEY_X2)
178                 return bank ? 2 : 0;
179
180         return bank;
181 }
182
183 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam,
184                            int blkaddr, int index)
185 {
186         int bank = npc_get_bank(mcam, index);
187         u64 cfg;
188
189         index &= (mcam->banksize - 1);
190         cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank));
191         return (cfg & 1);
192 }
193
194 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
195                            int blkaddr, int index, bool enable)
196 {
197         int bank = npc_get_bank(mcam, index);
198         int actbank = bank;
199
200         index &= (mcam->banksize - 1);
201         for (; bank < (actbank + mcam->banks_per_entry); bank++) {
202                 rvu_write64(rvu, blkaddr,
203                             NPC_AF_MCAMEX_BANKX_CFG(index, bank),
204                             enable ? 1 : 0);
205         }
206 }
207
208 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
209                                  int blkaddr, int index)
210 {
211         int bank = npc_get_bank(mcam, index);
212         int actbank = bank;
213
214         index &= (mcam->banksize - 1);
215         for (; bank < (actbank + mcam->banks_per_entry); bank++) {
216                 rvu_write64(rvu, blkaddr,
217                             NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0);
218                 rvu_write64(rvu, blkaddr,
219                             NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0);
220
221                 rvu_write64(rvu, blkaddr,
222                             NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0);
223                 rvu_write64(rvu, blkaddr,
224                             NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0);
225
226                 rvu_write64(rvu, blkaddr,
227                             NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0);
228                 rvu_write64(rvu, blkaddr,
229                             NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0);
230         }
231 }
232
233 static void npc_get_keyword(struct mcam_entry *entry, int idx,
234                             u64 *cam0, u64 *cam1)
235 {
236         u64 kw_mask = 0x00;
237
238 #define CAM_MASK(n)     (BIT_ULL(n) - 1)
239
240         /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and
241          * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1.
242          *
243          * Also, only 48 bits of BANKX_CAMX_W1 are valid.
244          */
245         switch (idx) {
246         case 0:
247                 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */
248                 *cam1 = entry->kw[0];
249                 kw_mask = entry->kw_mask[0];
250                 break;
251         case 1:
252                 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */
253                 *cam1 = entry->kw[1] & CAM_MASK(48);
254                 kw_mask = entry->kw_mask[1] & CAM_MASK(48);
255                 break;
256         case 2:
257                 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48>
258                  * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0>
259                  */
260                 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16);
261                 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16);
262                 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16);
263                 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16);
264                 break;
265         case 3:
266                 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48>
267                  * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0>
268                  */
269                 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16);
270                 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16);
271                 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16);
272                 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16);
273                 break;
274         case 4:
275                 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32>
276                  * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0>
277                  */
278                 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32);
279                 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32);
280                 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32);
281                 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32);
282                 break;
283         case 5:
284                 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32>
285                  * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0>
286                  */
287                 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32);
288                 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32);
289                 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32);
290                 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32);
291                 break;
292         case 6:
293                 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16>
294                  * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0>
295                  */
296                 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48);
297                 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48);
298                 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48);
299                 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48);
300                 break;
301         case 7:
302                 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */
303                 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48);
304                 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48);
305                 break;
306         }
307
308         *cam1 &= kw_mask;
309         *cam0 = ~*cam1 & kw_mask;
310 }
311
312 static void npc_fill_entryword(struct mcam_entry *entry, int idx,
313                                u64 cam0, u64 cam1)
314 {
315         /* Similar to npc_get_keyword, but fills mcam_entry structure from
316          * CAM registers.
317          */
318         switch (idx) {
319         case 0:
320                 entry->kw[0] = cam1;
321                 entry->kw_mask[0] = cam1 ^ cam0;
322                 break;
323         case 1:
324                 entry->kw[1] = cam1;
325                 entry->kw_mask[1] = cam1 ^ cam0;
326                 break;
327         case 2:
328                 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48;
329                 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48);
330                 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48;
331                 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48);
332                 break;
333         case 3:
334                 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48;
335                 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32);
336                 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48;
337                 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32);
338                 break;
339         case 4:
340                 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32;
341                 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32);
342                 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32;
343                 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32);
344                 break;
345         case 5:
346                 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32;
347                 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16);
348                 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32;
349                 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16);
350                 break;
351         case 6:
352                 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16;
353                 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16);
354                 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16;
355                 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16);
356                 break;
357         case 7:
358                 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16;
359                 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16;
360                 break;
361         }
362 }
363
364 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam,
365                                         int blkaddr, u16 pf_func)
366 {
367         int bank, nixlf, index;
368
369         /* get ucast entry rule entry index */
370         if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) {
371                 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n",
372                         __func__, pf_func);
373                 /* Action 0 is drop */
374                 return 0;
375         }
376
377         index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf,
378                                          NIXLF_UCAST_ENTRY);
379         bank = npc_get_bank(mcam, index);
380         index &= (mcam->banksize - 1);
381
382         return rvu_read64(rvu, blkaddr,
383                           NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
384 }
385
386 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam,
387                               int blkaddr, int index, struct mcam_entry *entry,
388                               bool *enable)
389 {
390         struct rvu_npc_mcam_rule *rule;
391         u16 owner, target_func;
392         struct rvu_pfvf *pfvf;
393         u64 rx_action;
394
395         owner = mcam->entry2pfvf_map[index];
396         target_func = (entry->action >> 4) & 0xffff;
397         /* do nothing when target is LBK/PF or owner is not PF */
398         if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) ||
399             (owner & RVU_PFVF_FUNC_MASK) ||
400             !(target_func & RVU_PFVF_FUNC_MASK))
401                 return;
402
403         /* save entry2target_pffunc */
404         pfvf = rvu_get_pfvf(rvu, target_func);
405         mcam->entry2target_pffunc[index] = target_func;
406
407         /* don't enable rule when nixlf not attached or initialized */
408         if (!(is_nixlf_attached(rvu, target_func) &&
409               test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
410                 *enable = false;
411
412         /* fix up not needed for the rules added by user(ntuple filters) */
413         list_for_each_entry(rule, &mcam->mcam_rules, list) {
414                 if (rule->entry == index)
415                         return;
416         }
417
418         /* AF modifies given action iff PF/VF has requested for it */
419         if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT)
420                 return;
421
422         /* copy VF default entry action to the VF mcam entry */
423         rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr,
424                                                  target_func);
425         if (rx_action)
426                 entry->action = rx_action;
427 }
428
429 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
430                                   int blkaddr, int index, u8 intf,
431                                   struct mcam_entry *entry, bool enable)
432 {
433         int bank = npc_get_bank(mcam, index);
434         int kw = 0, actbank, actindex;
435         u8 tx_intf_mask = ~intf & 0x3;
436         u8 tx_intf = intf;
437         u64 cam0, cam1;
438
439         actbank = bank; /* Save bank id, to set action later on */
440         actindex = index;
441         index &= (mcam->banksize - 1);
442
443         /* Disable before mcam entry update */
444         npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false);
445
446         /* Clear mcam entry to avoid writes being suppressed by NPC */
447         npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex);
448
449         /* CAM1 takes the comparison value and
450          * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'.
451          * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0
452          * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1
453          * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare.
454          */
455         for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) {
456                 /* Interface should be set in all banks */
457                 if (is_npc_intf_tx(intf)) {
458                         /* Last bit must be set and rest don't care
459                          * for TX interfaces
460                          */
461                         tx_intf_mask = 0x1;
462                         tx_intf = intf & tx_intf_mask;
463                         tx_intf_mask = ~tx_intf & tx_intf_mask;
464                 }
465
466                 rvu_write64(rvu, blkaddr,
467                             NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1),
468                             tx_intf);
469                 rvu_write64(rvu, blkaddr,
470                             NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0),
471                             tx_intf_mask);
472
473                 /* Set the match key */
474                 npc_get_keyword(entry, kw, &cam0, &cam1);
475                 rvu_write64(rvu, blkaddr,
476                             NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1);
477                 rvu_write64(rvu, blkaddr,
478                             NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0);
479
480                 npc_get_keyword(entry, kw + 1, &cam0, &cam1);
481                 rvu_write64(rvu, blkaddr,
482                             NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1);
483                 rvu_write64(rvu, blkaddr,
484                             NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0);
485         }
486
487         /* PF installing VF rule */
488         if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries)
489                 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable);
490
491         /* Set 'action' */
492         rvu_write64(rvu, blkaddr,
493                     NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action);
494
495         /* Set TAG 'action' */
496         rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank),
497                     entry->vtag_action);
498
499         /* Enable the entry */
500         if (enable)
501                 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true);
502 }
503
504 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
505                          int blkaddr, u16 src,
506                          struct mcam_entry *entry, u8 *intf, u8 *ena)
507 {
508         int sbank = npc_get_bank(mcam, src);
509         int bank, kw = 0;
510         u64 cam0, cam1;
511
512         src &= (mcam->banksize - 1);
513         bank = sbank;
514
515         for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) {
516                 cam1 = rvu_read64(rvu, blkaddr,
517                                   NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1));
518                 cam0 = rvu_read64(rvu, blkaddr,
519                                   NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0));
520                 npc_fill_entryword(entry, kw, cam0, cam1);
521
522                 cam1 = rvu_read64(rvu, blkaddr,
523                                   NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1));
524                 cam0 = rvu_read64(rvu, blkaddr,
525                                   NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0));
526                 npc_fill_entryword(entry, kw + 1, cam0, cam1);
527         }
528
529         entry->action = rvu_read64(rvu, blkaddr,
530                                    NPC_AF_MCAMEX_BANKX_ACTION(src, sbank));
531         entry->vtag_action =
532                 rvu_read64(rvu, blkaddr,
533                            NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank));
534         *intf = rvu_read64(rvu, blkaddr,
535                            NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3;
536         *ena = rvu_read64(rvu, blkaddr,
537                           NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1;
538 }
539
540 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,
541                                 int blkaddr, u16 src, u16 dest)
542 {
543         int dbank = npc_get_bank(mcam, dest);
544         int sbank = npc_get_bank(mcam, src);
545         u64 cfg, sreg, dreg;
546         int bank, i;
547
548         src &= (mcam->banksize - 1);
549         dest &= (mcam->banksize - 1);
550
551         /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */
552         for (bank = 0; bank < mcam->banks_per_entry; bank++) {
553                 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0);
554                 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0);
555                 for (i = 0; i < 6; i++) {
556                         cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8));
557                         rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg);
558                 }
559         }
560
561         /* Copy action */
562         cfg = rvu_read64(rvu, blkaddr,
563                          NPC_AF_MCAMEX_BANKX_ACTION(src, sbank));
564         rvu_write64(rvu, blkaddr,
565                     NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg);
566
567         /* Copy TAG action */
568         cfg = rvu_read64(rvu, blkaddr,
569                          NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank));
570         rvu_write64(rvu, blkaddr,
571                     NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg);
572
573         /* Enable or disable */
574         cfg = rvu_read64(rvu, blkaddr,
575                          NPC_AF_MCAMEX_BANKX_CFG(src, sbank));
576         rvu_write64(rvu, blkaddr,
577                     NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg);
578 }
579
580 u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
581                         int blkaddr, int index)
582 {
583         int bank = npc_get_bank(mcam, index);
584
585         index &= (mcam->banksize - 1);
586         return rvu_read64(rvu, blkaddr,
587                           NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
588 }
589
590 void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,
591                          int blkaddr, int index, u64 cfg)
592 {
593         int bank = npc_get_bank(mcam, index);
594
595         index &= (mcam->banksize - 1);
596         return rvu_write64(rvu, blkaddr,
597                            NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg);
598 }
599
600 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,
601                                  int nixlf, u64 chan, u8 *mac_addr)
602 {
603         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
604         struct npc_install_flow_req req = { 0 };
605         struct npc_install_flow_rsp rsp = { 0 };
606         struct npc_mcam *mcam = &rvu->hw->mcam;
607         struct nix_rx_action action = { 0 };
608         int blkaddr, index;
609
610         /* AF's and SDP VFs work in promiscuous mode */
611         if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc))
612                 return;
613
614         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
615         if (blkaddr < 0)
616                 return;
617
618         /* Ucast rule should not be installed if DMAC
619          * extraction is not supported by the profile.
620          */
621         if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf))
622                 return;
623
624         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
625                                          nixlf, NIXLF_UCAST_ENTRY);
626
627         /* Don't change the action if entry is already enabled
628          * Otherwise RSS action may get overwritten.
629          */
630         if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) {
631                 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,
632                                                       blkaddr, index);
633         } else {
634                 action.op = NIX_RX_ACTIONOP_UCAST;
635                 action.pf_func = pcifunc;
636         }
637
638         req.default_rule = 1;
639         ether_addr_copy(req.packet.dmac, mac_addr);
640         eth_broadcast_addr((u8 *)&req.mask.dmac);
641         req.features = BIT_ULL(NPC_DMAC);
642         req.channel = chan;
643         req.chan_mask = 0xFFFU;
644         req.intf = pfvf->nix_rx_intf;
645         req.op = action.op;
646         req.hdr.pcifunc = 0; /* AF is requester */
647         req.vf = action.pf_func;
648         req.index = action.index;
649         req.match_id = action.match_id;
650         req.flow_key_alg = action.flow_key_alg;
651
652         rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
653 }
654
655 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,
656                                    int nixlf, u64 chan, u8 chan_cnt)
657 {
658         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
659         struct npc_install_flow_req req = { 0 };
660         struct npc_install_flow_rsp rsp = { 0 };
661         struct npc_mcam *mcam = &rvu->hw->mcam;
662         struct rvu_hwinfo *hw = rvu->hw;
663         int blkaddr, ucast_idx, index;
664         struct nix_rx_action action = { 0 };
665         u64 relaxed_mask;
666         u8 flow_key_alg;
667
668         if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc))
669                 return;
670
671         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
672         if (blkaddr < 0)
673                 return;
674
675         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
676                                          nixlf, NIXLF_PROMISC_ENTRY);
677
678         if (is_cgx_vf(rvu, pcifunc))
679                 index = npc_get_nixlf_mcam_index(mcam,
680                                                  pcifunc & ~RVU_PFVF_FUNC_MASK,
681                                                  nixlf, NIXLF_PROMISC_ENTRY);
682
683         /* If the corresponding PF's ucast action is RSS,
684          * use the same action for promisc also
685          */
686         ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
687                                              nixlf, NIXLF_UCAST_ENTRY);
688         if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))
689                 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,
690                                                       blkaddr, ucast_idx);
691
692         if (action.op != NIX_RX_ACTIONOP_RSS) {
693                 *(u64 *)&action = 0;
694                 action.op = NIX_RX_ACTIONOP_UCAST;
695         }
696
697         flow_key_alg = action.flow_key_alg;
698
699         /* RX_ACTION set to MCAST for CGX PF's */
700         if (hw->cap.nix_rx_multicast && pfvf->use_mce_list &&
701             is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) {
702                 *(u64 *)&action = 0;
703                 action.op = NIX_RX_ACTIONOP_MCAST;
704                 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
705                 action.index = pfvf->promisc_mce_idx;
706         }
707
708         /* For cn10k the upper two bits of the channel number are
709          * cpt channel number. with masking out these bits in the
710          * mcam entry, same entry used for NIX will allow packets
711          * received from cpt for parsing.
712          */
713         if (!is_rvu_otx2(rvu)) {
714                 req.chan_mask = NIX_CHAN_CPT_X2P_MASK;
715         } else {
716                 req.chan_mask = 0xFFFU;
717         }
718
719         if (chan_cnt > 1) {
720                 if (!is_power_of_2(chan_cnt)) {
721                         dev_err(rvu->dev,
722                                 "%s: channel count more than 1, must be power of 2\n", __func__);
723                         return;
724                 }
725                 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1,
726                                            ilog2(chan_cnt));
727                 req.chan_mask &= relaxed_mask;
728         }
729
730         req.channel = chan;
731         req.intf = pfvf->nix_rx_intf;
732         req.entry = index;
733         req.op = action.op;
734         req.hdr.pcifunc = 0; /* AF is requester */
735         req.vf = pcifunc;
736         req.index = action.index;
737         req.match_id = action.match_id;
738         req.flow_key_alg = flow_key_alg;
739
740         rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
741 }
742
743 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc,
744                                   int nixlf, bool enable)
745 {
746         struct npc_mcam *mcam = &rvu->hw->mcam;
747         int blkaddr, index;
748
749         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
750         if (blkaddr < 0)
751                 return;
752
753         /* Get 'pcifunc' of PF device */
754         pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;
755
756         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
757                                          nixlf, NIXLF_PROMISC_ENTRY);
758         npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
759 }
760
761 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc,
762                                        int nixlf, u64 chan)
763 {
764         struct rvu_pfvf *pfvf;
765         struct npc_install_flow_req req = { 0 };
766         struct npc_install_flow_rsp rsp = { 0 };
767         struct npc_mcam *mcam = &rvu->hw->mcam;
768         struct rvu_hwinfo *hw = rvu->hw;
769         int blkaddr, index;
770
771         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
772         if (blkaddr < 0)
773                 return;
774
775         /* Skip LBK VFs */
776         if (is_lbk_vf(rvu, pcifunc))
777                 return;
778
779         /* If pkt replication is not supported,
780          * then only PF is allowed to add a bcast match entry.
781          */
782         if (!hw->cap.nix_rx_multicast && is_vf(pcifunc))
783                 return;
784
785         /* Get 'pcifunc' of PF device */
786         pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;
787         pfvf = rvu_get_pfvf(rvu, pcifunc);
788
789         /* Bcast rule should not be installed if both DMAC
790          * and LXMB extraction is not supported by the profile.
791          */
792         if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) &&
793             !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf))
794                 return;
795
796         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
797                                          nixlf, NIXLF_BCAST_ENTRY);
798
799         if (!hw->cap.nix_rx_multicast) {
800                 /* Early silicon doesn't support pkt replication,
801                  * so install entry with UCAST action, so that PF
802                  * receives all broadcast packets.
803                  */
804                 req.op = NIX_RX_ACTIONOP_UCAST;
805         } else {
806                 req.op = NIX_RX_ACTIONOP_MCAST;
807                 req.index = pfvf->bcast_mce_idx;
808         }
809
810         eth_broadcast_addr((u8 *)&req.packet.dmac);
811         eth_broadcast_addr((u8 *)&req.mask.dmac);
812         req.features = BIT_ULL(NPC_DMAC);
813         req.channel = chan;
814         req.chan_mask = 0xFFFU;
815         req.intf = pfvf->nix_rx_intf;
816         req.entry = index;
817         req.hdr.pcifunc = 0; /* AF is requester */
818         req.vf = pcifunc;
819
820         rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
821 }
822
823 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
824                                     u64 chan)
825 {
826         struct npc_install_flow_req req = { 0 };
827         struct npc_install_flow_rsp rsp = { 0 };
828         struct npc_mcam *mcam = &rvu->hw->mcam;
829         struct rvu_hwinfo *hw = rvu->hw;
830         int blkaddr, ucast_idx, index;
831         u8 mac_addr[ETH_ALEN] = { 0 };
832         struct nix_rx_action action = { 0 };
833         struct rvu_pfvf *pfvf;
834         u8 flow_key_alg;
835         u16 vf_func;
836
837         /* Only CGX PF/VF can add allmulticast entry */
838         if (is_lbk_vf(rvu, pcifunc) && is_sdp_vf(rvu, pcifunc))
839                 return;
840
841         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
842         if (blkaddr < 0)
843                 return;
844
845         /* Get 'pcifunc' of PF device */
846         vf_func = pcifunc & RVU_PFVF_FUNC_MASK;
847         pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;
848         pfvf = rvu_get_pfvf(rvu, pcifunc);
849
850         /* Mcast rule should not be installed if both DMAC
851          * and LXMB extraction is not supported by the profile.
852          */
853         if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) &&
854             !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf))
855                 return;
856
857         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
858                                          nixlf, NIXLF_ALLMULTI_ENTRY);
859
860         /* If the corresponding PF's ucast action is RSS,
861          * use the same action for multicast entry also
862          */
863         ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
864                                              nixlf, NIXLF_UCAST_ENTRY);
865         if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))
866                 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,
867                                                         blkaddr, ucast_idx);
868
869         flow_key_alg = action.flow_key_alg;
870         if (action.op != NIX_RX_ACTIONOP_RSS) {
871                 *(u64 *)&action = 0;
872                 action.op = NIX_RX_ACTIONOP_UCAST;
873                 action.pf_func = pcifunc;
874         }
875
876         /* RX_ACTION set to MCAST for CGX PF's */
877         if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) {
878                 *(u64 *)&action = 0;
879                 action.op = NIX_RX_ACTIONOP_MCAST;
880                 action.index = pfvf->mcast_mce_idx;
881         }
882
883         mac_addr[0] = 0x01;     /* LSB bit of 1st byte in DMAC */
884         ether_addr_copy(req.packet.dmac, mac_addr);
885         ether_addr_copy(req.mask.dmac, mac_addr);
886         req.features = BIT_ULL(NPC_DMAC);
887
888         /* For cn10k the upper two bits of the channel number are
889          * cpt channel number. with masking out these bits in the
890          * mcam entry, same entry used for NIX will allow packets
891          * received from cpt for parsing.
892          */
893         if (!is_rvu_otx2(rvu))
894                 req.chan_mask = NIX_CHAN_CPT_X2P_MASK;
895         else
896                 req.chan_mask = 0xFFFU;
897
898         req.channel = chan;
899         req.intf = pfvf->nix_rx_intf;
900         req.entry = index;
901         req.op = action.op;
902         req.hdr.pcifunc = 0; /* AF is requester */
903         req.vf = pcifunc | vf_func;
904         req.index = action.index;
905         req.match_id = action.match_id;
906         req.flow_key_alg = flow_key_alg;
907
908         rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);
909 }
910
911 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,
912                                    bool enable)
913 {
914         struct npc_mcam *mcam = &rvu->hw->mcam;
915         int blkaddr, index;
916
917         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
918         if (blkaddr < 0)
919                 return;
920
921         /* Get 'pcifunc' of PF device */
922         pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;
923
924         index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,
925                                          NIXLF_ALLMULTI_ENTRY);
926         npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
927 }
928
929 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam,
930                                      int blkaddr, u16 pcifunc, u64 rx_action)
931 {
932         int actindex, index, bank, entry;
933         struct rvu_npc_mcam_rule *rule;
934         bool enable, update;
935
936         if (!(pcifunc & RVU_PFVF_FUNC_MASK))
937                 return;
938
939         mutex_lock(&mcam->lock);
940         for (index = 0; index < mcam->bmap_entries; index++) {
941                 if (mcam->entry2target_pffunc[index] == pcifunc) {
942                         update = true;
943                         /* update not needed for the rules added via ntuple filters */
944                         list_for_each_entry(rule, &mcam->mcam_rules, list) {
945                                 if (rule->entry == index)
946                                         update = false;
947                         }
948                         if (!update)
949                                 continue;
950                         bank = npc_get_bank(mcam, index);
951                         actindex = index;
952                         entry = index & (mcam->banksize - 1);
953
954                         /* read vf flow entry enable status */
955                         enable = is_mcam_entry_enabled(rvu, mcam, blkaddr,
956                                                        actindex);
957                         /* disable before mcam entry update */
958                         npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex,
959                                               false);
960                         /* update 'action' */
961                         rvu_write64(rvu, blkaddr,
962                                     NPC_AF_MCAMEX_BANKX_ACTION(entry, bank),
963                                     rx_action);
964                         if (enable)
965                                 npc_enable_mcam_entry(rvu, mcam, blkaddr,
966                                                       actindex, true);
967                 }
968         }
969         mutex_unlock(&mcam->lock);
970 }
971
972 static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action,
973                                               struct rvu_pfvf *pfvf, int mcam_index, int blkaddr,
974                                               int alg_idx)
975
976 {
977         struct npc_mcam *mcam = &rvu->hw->mcam;
978         struct rvu_hwinfo *hw = rvu->hw;
979         int bank, op_rss;
980
981         if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index))
982                 return;
983
984         op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list);
985
986         bank = npc_get_bank(mcam, mcam_index);
987         mcam_index &= (mcam->banksize - 1);
988
989         /* If Rx action is MCAST update only RSS algorithm index */
990         if (!op_rss) {
991                 *(u64 *)&action = rvu_read64(rvu, blkaddr,
992                                 NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank));
993
994                 action.flow_key_alg = alg_idx;
995         }
996         rvu_write64(rvu, blkaddr,
997                     NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action);
998 }
999
1000 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,
1001                                     int group, int alg_idx, int mcam_index)
1002 {
1003         struct npc_mcam *mcam = &rvu->hw->mcam;
1004         struct nix_rx_action action;
1005         int blkaddr, index, bank;
1006         struct rvu_pfvf *pfvf;
1007
1008         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1009         if (blkaddr < 0)
1010                 return;
1011
1012         /* Check if this is for reserved default entry */
1013         if (mcam_index < 0) {
1014                 if (group != DEFAULT_RSS_CONTEXT_GROUP)
1015                         return;
1016                 index = npc_get_nixlf_mcam_index(mcam, pcifunc,
1017                                                  nixlf, NIXLF_UCAST_ENTRY);
1018         } else {
1019                 /* TODO: validate this mcam index */
1020                 index = mcam_index;
1021         }
1022
1023         if (index >= mcam->total_entries)
1024                 return;
1025
1026         bank = npc_get_bank(mcam, index);
1027         index &= (mcam->banksize - 1);
1028
1029         *(u64 *)&action = rvu_read64(rvu, blkaddr,
1030                                      NPC_AF_MCAMEX_BANKX_ACTION(index, bank));
1031         /* Ignore if no action was set earlier */
1032         if (!*(u64 *)&action)
1033                 return;
1034
1035         action.op = NIX_RX_ACTIONOP_RSS;
1036         action.pf_func = pcifunc;
1037         action.index = group;
1038         action.flow_key_alg = alg_idx;
1039
1040         rvu_write64(rvu, blkaddr,
1041                     NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action);
1042
1043         /* update the VF flow rule action with the VF default entry action */
1044         if (mcam_index < 0)
1045                 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc,
1046                                          *(u64 *)&action);
1047
1048         /* update the action change in default rule */
1049         pfvf = rvu_get_pfvf(rvu, pcifunc);
1050         if (pfvf->def_ucast_rule)
1051                 pfvf->def_ucast_rule->rx_action = action;
1052
1053         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
1054                                          nixlf, NIXLF_PROMISC_ENTRY);
1055
1056         /* If PF's promiscuous entry is enabled,
1057          * Set RSS action for that entry as well
1058          */
1059         npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr,
1060                                           alg_idx);
1061
1062         index = npc_get_nixlf_mcam_index(mcam, pcifunc,
1063                                          nixlf, NIXLF_ALLMULTI_ENTRY);
1064         /* If PF's allmulti  entry is enabled,
1065          * Set RSS action for that entry as well
1066          */
1067         npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr,
1068                                           alg_idx);
1069 }
1070
1071 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc,
1072                                   int nixlf, int type, bool enable)
1073 {
1074         struct npc_mcam *mcam = &rvu->hw->mcam;
1075         struct rvu_hwinfo *hw = rvu->hw;
1076         struct nix_mce_list *mce_list;
1077         int index, blkaddr, mce_idx;
1078         struct rvu_pfvf *pfvf;
1079
1080         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1081         if (blkaddr < 0)
1082                 return;
1083
1084         index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK,
1085                                          nixlf, type);
1086
1087         /* disable MCAM entry when packet replication is not supported by hw */
1088         if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) {
1089                 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
1090                 return;
1091         }
1092
1093         /* return incase mce list is not enabled */
1094         pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);
1095         if (hw->cap.nix_rx_multicast && is_vf(pcifunc) &&
1096             type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list)
1097                 return;
1098
1099         nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx);
1100
1101         nix_update_mce_list(rvu, pcifunc, mce_list,
1102                             mce_idx, index, enable);
1103         if (enable)
1104                 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
1105 }
1106
1107 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc,
1108                                        int nixlf, bool enable)
1109 {
1110         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1111         struct npc_mcam *mcam = &rvu->hw->mcam;
1112         int index, blkaddr;
1113
1114         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1115         if (blkaddr < 0)
1116                 return;
1117
1118         /* Ucast MCAM match entry of this PF/VF */
1119         if (npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC),
1120                                      pfvf->nix_rx_intf)) {
1121                 index = npc_get_nixlf_mcam_index(mcam, pcifunc,
1122                                                  nixlf, NIXLF_UCAST_ENTRY);
1123                 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);
1124         }
1125
1126         /* Nothing to do for VFs, on platforms where pkt replication
1127          * is not supported
1128          */
1129         if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast)
1130                 return;
1131
1132         /* add/delete pf_func to broadcast MCE list */
1133         npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,
1134                                      NIXLF_BCAST_ENTRY, enable);
1135 }
1136
1137 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf)
1138 {
1139         if (nixlf < 0)
1140                 return;
1141
1142         npc_enadis_default_entries(rvu, pcifunc, nixlf, false);
1143
1144         /* Delete multicast and promisc MCAM entries */
1145         npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,
1146                                      NIXLF_ALLMULTI_ENTRY, false);
1147         npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,
1148                                      NIXLF_PROMISC_ENTRY, false);
1149 }
1150
1151 bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable)
1152 {
1153         int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1154         struct npc_mcam *mcam = &rvu->hw->mcam;
1155         struct rvu_npc_mcam_rule *rule, *tmp;
1156
1157         mutex_lock(&mcam->lock);
1158
1159         list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
1160                 if (rule->intf != intf)
1161                         continue;
1162
1163                 if (rule->entry != entry)
1164                         continue;
1165
1166                 rule->enable = enable;
1167                 mutex_unlock(&mcam->lock);
1168
1169                 npc_enable_mcam_entry(rvu, mcam, blkaddr,
1170                                       entry, enable);
1171
1172                 return true;
1173         }
1174
1175         mutex_unlock(&mcam->lock);
1176         return false;
1177 }
1178
1179 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf)
1180 {
1181         if (nixlf < 0)
1182                 return;
1183
1184         /* Enables only broadcast match entry. Promisc/Allmulti are enabled
1185          * in set_rx_mode mbox handler.
1186          */
1187         npc_enadis_default_entries(rvu, pcifunc, nixlf, true);
1188 }
1189
1190 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf)
1191 {
1192         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
1193         struct npc_mcam *mcam = &rvu->hw->mcam;
1194         struct rvu_npc_mcam_rule *rule, *tmp;
1195         int blkaddr;
1196
1197         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1198         if (blkaddr < 0)
1199                 return;
1200
1201         mutex_lock(&mcam->lock);
1202
1203         /* Disable MCAM entries directing traffic to this 'pcifunc' */
1204         list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
1205                 if (is_npc_intf_rx(rule->intf) &&
1206                     rule->rx_action.pf_func == pcifunc &&
1207                     rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) {
1208                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1209                                               rule->entry, false);
1210                         rule->enable = false;
1211                         /* Indicate that default rule is disabled */
1212                         if (rule->default_rule) {
1213                                 pfvf->def_ucast_rule = NULL;
1214                                 list_del(&rule->list);
1215                                 kfree(rule);
1216                         }
1217                 }
1218         }
1219
1220         mutex_unlock(&mcam->lock);
1221
1222         npc_mcam_disable_flows(rvu, pcifunc);
1223
1224         rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
1225 }
1226
1227 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf)
1228 {
1229         struct npc_mcam *mcam = &rvu->hw->mcam;
1230         struct rvu_npc_mcam_rule *rule, *tmp;
1231         int blkaddr;
1232
1233         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1234         if (blkaddr < 0)
1235                 return;
1236
1237         mutex_lock(&mcam->lock);
1238
1239         /* Free all MCAM entries owned by this 'pcifunc' */
1240         npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc);
1241
1242         /* Free all MCAM counters owned by this 'pcifunc' */
1243         npc_mcam_free_all_counters(rvu, mcam, pcifunc);
1244
1245         /* Delete MCAM entries owned by this 'pcifunc' */
1246         list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
1247                 if (rule->owner == pcifunc && !rule->default_rule) {
1248                         list_del(&rule->list);
1249                         kfree(rule);
1250                 }
1251         }
1252
1253         mutex_unlock(&mcam->lock);
1254
1255         rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);
1256 }
1257
1258 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr,
1259                                 struct npc_mcam_kex *mkex, u8 intf)
1260 {
1261         int lid, lt, ld, fl;
1262
1263         if (is_npc_intf_tx(intf))
1264                 return;
1265
1266         rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
1267                     mkex->keyx_cfg[NIX_INTF_RX]);
1268
1269         /* Program LDATA */
1270         for (lid = 0; lid < NPC_MAX_LID; lid++) {
1271                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
1272                         for (ld = 0; ld < NPC_MAX_LD; ld++)
1273                                 SET_KEX_LD(intf, lid, lt, ld,
1274                                            mkex->intf_lid_lt_ld[NIX_INTF_RX]
1275                                            [lid][lt][ld]);
1276                 }
1277         }
1278         /* Program LFLAGS */
1279         for (ld = 0; ld < NPC_MAX_LD; ld++) {
1280                 for (fl = 0; fl < NPC_MAX_LFL; fl++)
1281                         SET_KEX_LDFLAGS(intf, ld, fl,
1282                                         mkex->intf_ld_flags[NIX_INTF_RX]
1283                                         [ld][fl]);
1284         }
1285 }
1286
1287 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr,
1288                                 struct npc_mcam_kex *mkex, u8 intf)
1289 {
1290         int lid, lt, ld, fl;
1291
1292         if (is_npc_intf_rx(intf))
1293                 return;
1294
1295         rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
1296                     mkex->keyx_cfg[NIX_INTF_TX]);
1297
1298         /* Program LDATA */
1299         for (lid = 0; lid < NPC_MAX_LID; lid++) {
1300                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
1301                         for (ld = 0; ld < NPC_MAX_LD; ld++)
1302                                 SET_KEX_LD(intf, lid, lt, ld,
1303                                            mkex->intf_lid_lt_ld[NIX_INTF_TX]
1304                                            [lid][lt][ld]);
1305                 }
1306         }
1307         /* Program LFLAGS */
1308         for (ld = 0; ld < NPC_MAX_LD; ld++) {
1309                 for (fl = 0; fl < NPC_MAX_LFL; fl++)
1310                         SET_KEX_LDFLAGS(intf, ld, fl,
1311                                         mkex->intf_ld_flags[NIX_INTF_TX]
1312                                         [ld][fl]);
1313         }
1314 }
1315
1316 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,
1317                                      struct npc_mcam_kex *mkex)
1318 {
1319         struct rvu_hwinfo *hw = rvu->hw;
1320         u8 intf;
1321         int ld;
1322
1323         for (ld = 0; ld < NPC_MAX_LD; ld++)
1324                 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld),
1325                             mkex->kex_ld_flags[ld]);
1326
1327         for (intf = 0; intf < hw->npc_intfs; intf++) {
1328                 npc_program_mkex_rx(rvu, blkaddr, mkex, intf);
1329                 npc_program_mkex_tx(rvu, blkaddr, mkex, intf);
1330         }
1331
1332         /* Programme mkex hash profile */
1333         npc_program_mkex_hash(rvu, blkaddr);
1334 }
1335
1336 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr,
1337                                  u64 *size)
1338 {
1339         u64 prfl_addr, prfl_sz;
1340
1341         if (!rvu->fwdata)
1342                 return -EINVAL;
1343
1344         prfl_addr = rvu->fwdata->mcam_addr;
1345         prfl_sz = rvu->fwdata->mcam_sz;
1346
1347         if (!prfl_addr || !prfl_sz)
1348                 return -EINVAL;
1349
1350         *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz);
1351         if (!(*prfl_img_addr))
1352                 return -ENOMEM;
1353
1354         *size = prfl_sz;
1355
1356         return 0;
1357 }
1358
1359 /* strtoull of "mkexprof" with base:36 */
1360 #define MKEX_END_SIGN  0xdeadbeef
1361
1362 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr,
1363                                   const char *mkex_profile)
1364 {
1365         struct device *dev = &rvu->pdev->dev;
1366         struct npc_mcam_kex *mcam_kex;
1367         void __iomem *mkex_prfl_addr = NULL;
1368         u64 prfl_sz;
1369         int ret;
1370
1371         /* If user not selected mkex profile */
1372         if (rvu->kpu_fwdata_sz ||
1373             !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN))
1374                 goto program_mkex;
1375
1376         /* Setting up the mapping for mkex profile image */
1377         ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz);
1378         if (ret < 0)
1379                 goto program_mkex;
1380
1381         mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr;
1382
1383         while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) {
1384                 /* Compare with mkex mod_param name string */
1385                 if (mcam_kex->mkex_sign == MKEX_SIGN &&
1386                     !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) {
1387                         /* Due to an errata (35786) in A0/B0 pass silicon,
1388                          * parse nibble enable configuration has to be
1389                          * identical for both Rx and Tx interfaces.
1390                          */
1391                         if (!is_rvu_96xx_B0(rvu) ||
1392                             mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX])
1393                                 rvu->kpu.mkex = mcam_kex;
1394                         goto program_mkex;
1395                 }
1396
1397                 mcam_kex++;
1398                 prfl_sz -= sizeof(struct npc_mcam_kex);
1399         }
1400         dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);
1401
1402 program_mkex:
1403         dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name);
1404         /* Program selected mkex profile */
1405         npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex);
1406         if (mkex_prfl_addr)
1407                 iounmap(mkex_prfl_addr);
1408 }
1409
1410 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr,
1411                                  const struct npc_kpu_profile_action *kpuaction,
1412                                  int kpu, int entry, bool pkind)
1413 {
1414         struct npc_kpu_action0 action0 = {0};
1415         struct npc_kpu_action1 action1 = {0};
1416         u64 reg;
1417
1418         action1.errlev = kpuaction->errlev;
1419         action1.errcode = kpuaction->errcode;
1420         action1.dp0_offset = kpuaction->dp0_offset;
1421         action1.dp1_offset = kpuaction->dp1_offset;
1422         action1.dp2_offset = kpuaction->dp2_offset;
1423
1424         if (pkind)
1425                 reg = NPC_AF_PKINDX_ACTION1(entry);
1426         else
1427                 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry);
1428
1429         rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1);
1430
1431         action0.byp_count = kpuaction->bypass_count;
1432         action0.capture_ena = kpuaction->cap_ena;
1433         action0.parse_done = kpuaction->parse_done;
1434         action0.next_state = kpuaction->next_state;
1435         action0.capture_lid = kpuaction->lid;
1436         action0.capture_ltype = kpuaction->ltype;
1437         action0.capture_flags = kpuaction->flags;
1438         action0.ptr_advance = kpuaction->ptr_advance;
1439         action0.var_len_offset = kpuaction->offset;
1440         action0.var_len_mask = kpuaction->mask;
1441         action0.var_len_right = kpuaction->right;
1442         action0.var_len_shift = kpuaction->shift;
1443
1444         if (pkind)
1445                 reg = NPC_AF_PKINDX_ACTION0(entry);
1446         else
1447                 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry);
1448
1449         rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0);
1450 }
1451
1452 static void npc_config_kpucam(struct rvu *rvu, int blkaddr,
1453                               const struct npc_kpu_profile_cam *kpucam,
1454                               int kpu, int entry)
1455 {
1456         struct npc_kpu_cam cam0 = {0};
1457         struct npc_kpu_cam cam1 = {0};
1458
1459         cam1.state = kpucam->state & kpucam->state_mask;
1460         cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask;
1461         cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask;
1462         cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask;
1463
1464         cam0.state = ~kpucam->state & kpucam->state_mask;
1465         cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask;
1466         cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask;
1467         cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask;
1468
1469         rvu_write64(rvu, blkaddr,
1470                     NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0);
1471         rvu_write64(rvu, blkaddr,
1472                     NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1);
1473 }
1474
1475 static inline u64 enable_mask(int count)
1476 {
1477         return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL));
1478 }
1479
1480 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu,
1481                                     const struct npc_kpu_profile *profile)
1482 {
1483         int entry, num_entries, max_entries;
1484         u64 entry_mask;
1485
1486         if (profile->cam_entries != profile->action_entries) {
1487                 dev_err(rvu->dev,
1488                         "KPU%d: CAM and action entries [%d != %d] not equal\n",
1489                         kpu, profile->cam_entries, profile->action_entries);
1490         }
1491
1492         max_entries = rvu->hw->npc_kpu_entries;
1493
1494         /* Program CAM match entries for previous KPU extracted data */
1495         num_entries = min_t(int, profile->cam_entries, max_entries);
1496         for (entry = 0; entry < num_entries; entry++)
1497                 npc_config_kpucam(rvu, blkaddr,
1498                                   &profile->cam[entry], kpu, entry);
1499
1500         /* Program this KPU's actions */
1501         num_entries = min_t(int, profile->action_entries, max_entries);
1502         for (entry = 0; entry < num_entries; entry++)
1503                 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry],
1504                                      kpu, entry, false);
1505
1506         /* Enable all programmed entries */
1507         num_entries = min_t(int, profile->action_entries, profile->cam_entries);
1508         entry_mask = enable_mask(num_entries);
1509         /* Disable first KPU_MAX_CST_ENT entries for built-in profile */
1510         if (!rvu->kpu.custom)
1511                 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0);
1512         rvu_write64(rvu, blkaddr,
1513                     NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask);
1514         if (num_entries > 64) {
1515                 rvu_write64(rvu, blkaddr,
1516                             NPC_AF_KPUX_ENTRY_DISX(kpu, 1),
1517                             enable_mask(num_entries - 64));
1518         }
1519
1520         /* Enable this KPU */
1521         rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01);
1522 }
1523
1524 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile)
1525 {
1526         profile->custom = 0;
1527         profile->name = def_pfl_name;
1528         profile->version = NPC_KPU_PROFILE_VER;
1529         profile->ikpu = ikpu_action_entries;
1530         profile->pkinds = ARRAY_SIZE(ikpu_action_entries);
1531         profile->kpu = npc_kpu_profiles;
1532         profile->kpus = ARRAY_SIZE(npc_kpu_profiles);
1533         profile->lt_def = &npc_lt_defaults;
1534         profile->mkex = &npc_mkex_default;
1535         profile->mkex_hash = &npc_mkex_hash_default;
1536
1537         return 0;
1538 }
1539
1540 static int npc_apply_custom_kpu(struct rvu *rvu,
1541                                 struct npc_kpu_profile_adapter *profile)
1542 {
1543         size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0;
1544         struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata;
1545         struct npc_kpu_profile_action *action;
1546         struct npc_kpu_profile_cam *cam;
1547         struct npc_kpu_fwdata *fw_kpu;
1548         int entries;
1549         u16 kpu, entry;
1550
1551         if (rvu->kpu_fwdata_sz < hdr_sz) {
1552                 dev_warn(rvu->dev, "Invalid KPU profile size\n");
1553                 return -EINVAL;
1554         }
1555         if (le64_to_cpu(fw->signature) != KPU_SIGN) {
1556                 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n",
1557                          fw->signature);
1558                 return -EINVAL;
1559         }
1560         /* Verify if the using known profile structure */
1561         if (NPC_KPU_VER_MAJ(profile->version) >
1562             NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) {
1563                 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n",
1564                          NPC_KPU_VER_MAJ(profile->version),
1565                          NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER));
1566                 return -EINVAL;
1567         }
1568         /* Verify if profile is aligned with the required kernel changes */
1569         if (NPC_KPU_VER_MIN(profile->version) <
1570             NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) {
1571                 dev_warn(rvu->dev,
1572                          "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n",
1573                          NPC_KPU_VER_MAJ(profile->version),
1574                          NPC_KPU_VER_MIN(profile->version),
1575                          NPC_KPU_VER_PATCH(profile->version),
1576                          NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER),
1577                          NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER),
1578                          NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER));
1579                 return -EINVAL;
1580         }
1581         /* Verify if profile fits the HW */
1582         if (fw->kpus > profile->kpus) {
1583                 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus,
1584                          profile->kpus);
1585                 return -EINVAL;
1586         }
1587
1588         profile->custom = 1;
1589         profile->name = fw->name;
1590         profile->version = le64_to_cpu(fw->version);
1591         profile->mkex = &fw->mkex;
1592         profile->lt_def = &fw->lt_def;
1593
1594         for (kpu = 0; kpu < fw->kpus; kpu++) {
1595                 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset);
1596                 if (fw_kpu->entries > KPU_MAX_CST_ENT)
1597                         dev_warn(rvu->dev,
1598                                  "Too many custom entries on KPU%d: %d > %d\n",
1599                                  kpu, fw_kpu->entries, KPU_MAX_CST_ENT);
1600                 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT);
1601                 cam = (struct npc_kpu_profile_cam *)fw_kpu->data;
1602                 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam);
1603                 action = (struct npc_kpu_profile_action *)(fw->data + offset);
1604                 offset += fw_kpu->entries * sizeof(*action);
1605                 if (rvu->kpu_fwdata_sz < hdr_sz + offset) {
1606                         dev_warn(rvu->dev,
1607                                  "Profile size mismatch on KPU%i parsing.\n",
1608                                  kpu + 1);
1609                         return -EINVAL;
1610                 }
1611                 for (entry = 0; entry < entries; entry++) {
1612                         profile->kpu[kpu].cam[entry] = cam[entry];
1613                         profile->kpu[kpu].action[entry] = action[entry];
1614                 }
1615         }
1616
1617         return 0;
1618 }
1619
1620 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr,
1621                                  u64 prfl_sz, const char *kpu_profile)
1622 {
1623         struct npc_kpu_profile_fwdata *kpu_data = NULL;
1624         int rc = -EINVAL;
1625
1626         kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr;
1627         if (le64_to_cpu(kpu_data->signature) == KPU_SIGN &&
1628             !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) {
1629                 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n",
1630                          kpu_profile);
1631                 rvu->kpu_fwdata = kpu_data;
1632                 rvu->kpu_fwdata_sz = prfl_sz;
1633                 rvu->kpu_prfl_addr = prfl_addr;
1634                 rc = 0;
1635         }
1636
1637         return rc;
1638 }
1639
1640 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz,
1641                                          const char *kpu_profile)
1642 {
1643         struct npc_coalesced_kpu_prfl *img_data = NULL;
1644         int i = 0, rc = -EINVAL;
1645         void __iomem *kpu_prfl_addr;
1646         u32 offset;
1647
1648         img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr;
1649         if (le64_to_cpu(img_data->signature) == KPU_SIGN &&
1650             !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) {
1651                 /* Loaded profile is a single KPU profile. */
1652                 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr,
1653                                            prfl_sz, kpu_profile);
1654                 goto done;
1655         }
1656
1657         /* Loaded profile is coalesced image, offset of first KPU profile.*/
1658         offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) +
1659                 (img_data->num_prfl * sizeof(uint16_t));
1660         /* Check if mapped image is coalesced image. */
1661         while (i < img_data->num_prfl) {
1662                 /* Profile image offsets are rounded up to next 8 multiple.*/
1663                 offset = ALIGN_8B_CEIL(offset);
1664                 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr +
1665                                          offset);
1666                 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr,
1667                                            img_data->prfl_sz[i], kpu_profile);
1668                 if (!rc)
1669                         break;
1670                 /* Calculating offset of profile image based on profile size.*/
1671                 offset += img_data->prfl_sz[i];
1672                 i++;
1673         }
1674 done:
1675         return rc;
1676 }
1677
1678 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile)
1679 {
1680         int ret = -EINVAL;
1681         u64 prfl_sz;
1682
1683         /* Setting up the mapping for NPC profile image */
1684         ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz);
1685         if (ret < 0)
1686                 goto done;
1687
1688         /* Detect if profile is coalesced or single KPU profile and load */
1689         ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile);
1690         if (ret == 0)
1691                 goto done;
1692
1693         /* Cleaning up if KPU profile image from fwdata is not valid. */
1694         if (rvu->kpu_prfl_addr) {
1695                 iounmap(rvu->kpu_prfl_addr);
1696                 rvu->kpu_prfl_addr = NULL;
1697                 rvu->kpu_fwdata_sz = 0;
1698                 rvu->kpu_fwdata = NULL;
1699         }
1700
1701 done:
1702         return ret;
1703 }
1704
1705 static void npc_load_kpu_profile(struct rvu *rvu)
1706 {
1707         struct npc_kpu_profile_adapter *profile = &rvu->kpu;
1708         const char *kpu_profile = rvu->kpu_pfl_name;
1709         const struct firmware *fw = NULL;
1710         bool retry_fwdb = false;
1711
1712         /* If user not specified profile customization */
1713         if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN))
1714                 goto revert_to_default;
1715         /* First prepare default KPU, then we'll customize top entries. */
1716         npc_prepare_default_kpu(profile);
1717
1718         /* Order of preceedence for load loading NPC profile (high to low)
1719          * Firmware binary in filesystem.
1720          * Firmware database method.
1721          * Default KPU profile.
1722          */
1723         if (!request_firmware_direct(&fw, kpu_profile, rvu->dev)) {
1724                 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n",
1725                          kpu_profile);
1726                 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL);
1727                 if (rvu->kpu_fwdata) {
1728                         memcpy(rvu->kpu_fwdata, fw->data, fw->size);
1729                         rvu->kpu_fwdata_sz = fw->size;
1730                 }
1731                 release_firmware(fw);
1732                 retry_fwdb = true;
1733                 goto program_kpu;
1734         }
1735
1736 load_image_fwdb:
1737         /* Loading the KPU profile using firmware database */
1738         if (npc_load_kpu_profile_fwdb(rvu, kpu_profile))
1739                 goto revert_to_default;
1740
1741 program_kpu:
1742         /* Apply profile customization if firmware was loaded. */
1743         if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) {
1744                 /* If image from firmware filesystem fails to load or invalid
1745                  * retry with firmware database method.
1746                  */
1747                 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) {
1748                         /* Loading image from firmware database failed. */
1749                         if (rvu->kpu_prfl_addr) {
1750                                 iounmap(rvu->kpu_prfl_addr);
1751                                 rvu->kpu_prfl_addr = NULL;
1752                         } else {
1753                                 kfree(rvu->kpu_fwdata);
1754                         }
1755                         rvu->kpu_fwdata = NULL;
1756                         rvu->kpu_fwdata_sz = 0;
1757                         if (retry_fwdb) {
1758                                 retry_fwdb = false;
1759                                 goto load_image_fwdb;
1760                         }
1761                 }
1762
1763                 dev_warn(rvu->dev,
1764                          "Can't load KPU profile %s. Using default.\n",
1765                          kpu_profile);
1766                 kfree(rvu->kpu_fwdata);
1767                 rvu->kpu_fwdata = NULL;
1768                 goto revert_to_default;
1769         }
1770
1771         dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n",
1772                  profile->name, NPC_KPU_VER_MAJ(profile->version),
1773                  NPC_KPU_VER_MIN(profile->version),
1774                  NPC_KPU_VER_PATCH(profile->version));
1775
1776         return;
1777
1778 revert_to_default:
1779         npc_prepare_default_kpu(profile);
1780 }
1781
1782 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr)
1783 {
1784         struct rvu_hwinfo *hw = rvu->hw;
1785         int num_pkinds, num_kpus, idx;
1786
1787         /* Disable all KPUs and their entries */
1788         for (idx = 0; idx < hw->npc_kpus; idx++) {
1789                 rvu_write64(rvu, blkaddr,
1790                             NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL);
1791                 rvu_write64(rvu, blkaddr,
1792                             NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL);
1793                 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00);
1794         }
1795
1796         /* Load and customize KPU profile. */
1797         npc_load_kpu_profile(rvu);
1798
1799         /* First program IKPU profile i.e PKIND configs.
1800          * Check HW max count to avoid configuring junk or
1801          * writing to unsupported CSR addresses.
1802          */
1803         num_pkinds = rvu->kpu.pkinds;
1804         num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds);
1805
1806         for (idx = 0; idx < num_pkinds; idx++)
1807                 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true);
1808
1809         /* Program KPU CAM and Action profiles */
1810         num_kpus = rvu->kpu.kpus;
1811         num_kpus = min_t(int, hw->npc_kpus, num_kpus);
1812
1813         for (idx = 0; idx < num_kpus; idx++)
1814                 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]);
1815 }
1816
1817 void npc_mcam_rsrcs_deinit(struct rvu *rvu)
1818 {
1819         struct npc_mcam *mcam = &rvu->hw->mcam;
1820
1821         bitmap_free(mcam->bmap);
1822         bitmap_free(mcam->bmap_reverse);
1823         kfree(mcam->entry2pfvf_map);
1824         kfree(mcam->cntr2pfvf_map);
1825         kfree(mcam->entry2cntr_map);
1826         kfree(mcam->cntr_refcnt);
1827         kfree(mcam->entry2target_pffunc);
1828         kfree(mcam->counters.bmap);
1829 }
1830
1831 int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr)
1832 {
1833         int nixlf_count = rvu_get_nixlf_count(rvu);
1834         struct npc_mcam *mcam = &rvu->hw->mcam;
1835         int rsvd, err;
1836         u16 index;
1837         int cntr;
1838         u64 cfg;
1839
1840         /* Actual number of MCAM entries vary by entry size */
1841         cfg = (rvu_read64(rvu, blkaddr,
1842                           NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07;
1843         mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize;
1844         mcam->keysize = cfg;
1845
1846         /* Number of banks combined per MCAM entry */
1847         if (cfg == NPC_MCAM_KEY_X4)
1848                 mcam->banks_per_entry = 4;
1849         else if (cfg == NPC_MCAM_KEY_X2)
1850                 mcam->banks_per_entry = 2;
1851         else
1852                 mcam->banks_per_entry = 1;
1853
1854         /* Reserve one MCAM entry for each of the NIX LF to
1855          * guarantee space to install default matching DMAC rule.
1856          * Also reserve 2 MCAM entries for each PF for default
1857          * channel based matching or 'bcast & promisc' matching to
1858          * support BCAST and PROMISC modes of operation for PFs.
1859          * PF0 is excluded.
1860          */
1861         rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) +
1862                 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF);
1863         if (mcam->total_entries <= rsvd) {
1864                 dev_warn(rvu->dev,
1865                          "Insufficient NPC MCAM size %d for pkt I/O, exiting\n",
1866                          mcam->total_entries);
1867                 return -ENOMEM;
1868         }
1869
1870         mcam->bmap_entries = mcam->total_entries - rsvd;
1871         mcam->nixlf_offset = mcam->bmap_entries;
1872         mcam->pf_offset = mcam->nixlf_offset + nixlf_count;
1873
1874         /* Allocate bitmaps for managing MCAM entries */
1875         mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL);
1876         if (!mcam->bmap)
1877                 return -ENOMEM;
1878
1879         mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL);
1880         if (!mcam->bmap_reverse)
1881                 goto free_bmap;
1882
1883         mcam->bmap_fcnt = mcam->bmap_entries;
1884
1885         /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */
1886         mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16),
1887                                        GFP_KERNEL);
1888
1889         if (!mcam->entry2pfvf_map)
1890                 goto free_bmap_reverse;
1891
1892         /* Reserve 1/8th of MCAM entries at the bottom for low priority
1893          * allocations and another 1/8th at the top for high priority
1894          * allocations.
1895          */
1896         mcam->lprio_count = mcam->bmap_entries / 8;
1897         if (mcam->lprio_count > BITS_PER_LONG)
1898                 mcam->lprio_count = round_down(mcam->lprio_count,
1899                                                BITS_PER_LONG);
1900         mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count;
1901         mcam->hprio_count = mcam->lprio_count;
1902         mcam->hprio_end = mcam->hprio_count;
1903
1904         /* Allocate bitmap for managing MCAM counters and memory
1905          * for saving counter to RVU PFFUNC allocation mapping.
1906          */
1907         err = rvu_alloc_bitmap(&mcam->counters);
1908         if (err)
1909                 goto free_entry_map;
1910
1911         mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16),
1912                                       GFP_KERNEL);
1913         if (!mcam->cntr2pfvf_map)
1914                 goto free_cntr_bmap;
1915
1916         /* Alloc memory for MCAM entry to counter mapping and for tracking
1917          * counter's reference count.
1918          */
1919         mcam->entry2cntr_map = kcalloc(mcam->bmap_entries, sizeof(u16),
1920                                        GFP_KERNEL);
1921         if (!mcam->entry2cntr_map)
1922                 goto free_cntr_map;
1923
1924         mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16),
1925                                     GFP_KERNEL);
1926         if (!mcam->cntr_refcnt)
1927                 goto free_entry_cntr_map;
1928
1929         /* Alloc memory for saving target device of mcam rule */
1930         mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries,
1931                                                   sizeof(u16), GFP_KERNEL);
1932         if (!mcam->entry2target_pffunc)
1933                 goto free_cntr_refcnt;
1934
1935         for (index = 0; index < mcam->bmap_entries; index++) {
1936                 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP;
1937                 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP;
1938         }
1939
1940         for (cntr = 0; cntr < mcam->counters.max; cntr++)
1941                 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP;
1942
1943         mutex_init(&mcam->lock);
1944
1945         return 0;
1946
1947 free_cntr_refcnt:
1948         kfree(mcam->cntr_refcnt);
1949 free_entry_cntr_map:
1950         kfree(mcam->entry2cntr_map);
1951 free_cntr_map:
1952         kfree(mcam->cntr2pfvf_map);
1953 free_cntr_bmap:
1954         kfree(mcam->counters.bmap);
1955 free_entry_map:
1956         kfree(mcam->entry2pfvf_map);
1957 free_bmap_reverse:
1958         bitmap_free(mcam->bmap_reverse);
1959 free_bmap:
1960         bitmap_free(mcam->bmap);
1961
1962         return -ENOMEM;
1963 }
1964
1965 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr)
1966 {
1967         struct npc_pkind *pkind = &rvu->hw->pkind;
1968         struct npc_mcam *mcam = &rvu->hw->mcam;
1969         struct rvu_hwinfo *hw = rvu->hw;
1970         u64 npc_const, npc_const1;
1971         u64 npc_const2 = 0;
1972
1973         npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST);
1974         npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1);
1975         if (npc_const1 & BIT_ULL(63))
1976                 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2);
1977
1978         pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT;
1979         hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL;
1980         hw->npc_kpu_entries = npc_const1 & 0xFFFULL;
1981         hw->npc_kpus = (npc_const >> 8) & 0x1FULL;
1982         hw->npc_intfs = npc_const & 0xFULL;
1983         hw->npc_counters = (npc_const >> 48) & 0xFFFFULL;
1984
1985         mcam->banks = (npc_const >> 44) & 0xFULL;
1986         mcam->banksize = (npc_const >> 28) & 0xFFFFULL;
1987         hw->npc_stat_ena = BIT_ULL(9);
1988         /* Extended set */
1989         if (npc_const2) {
1990                 hw->npc_ext_set = true;
1991                 /* 96xx supports only match_stats and npc_counters
1992                  * reflected in NPC_AF_CONST reg.
1993                  * STAT_SEL and ENA are at [0:8] and 9 bit positions.
1994                  * 98xx has both match_stat and ext and npc_counter
1995                  * reflected in NPC_AF_CONST2
1996                  * STAT_SEL_EXT added at [12:14] bit position.
1997                  * cn10k supports only ext and hence npc_counters in
1998                  * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2.
1999                  * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63
2000                  */
2001                 if (!hw->npc_counters)
2002                         hw->npc_stat_ena = BIT_ULL(63);
2003                 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL;
2004                 mcam->banksize = npc_const2 & 0xFFFFULL;
2005         }
2006
2007         mcam->counters.max = hw->npc_counters;
2008 }
2009
2010 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr)
2011 {
2012         struct npc_mcam_kex *mkex = rvu->kpu.mkex;
2013         struct npc_mcam *mcam = &rvu->hw->mcam;
2014         struct rvu_hwinfo *hw = rvu->hw;
2015         u64 nibble_ena, rx_kex, tx_kex;
2016         u8 intf;
2017
2018         /* Reserve last counter for MCAM RX miss action which is set to
2019          * drop packet. This way we will know how many pkts didn't match
2020          * any MCAM entry.
2021          */
2022         mcam->counters.max--;
2023         mcam->rx_miss_act_cntr = mcam->counters.max;
2024
2025         rx_kex = mkex->keyx_cfg[NIX_INTF_RX];
2026         tx_kex = mkex->keyx_cfg[NIX_INTF_TX];
2027         nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex);
2028
2029         nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena);
2030         if (nibble_ena) {
2031                 tx_kex &= ~NPC_PARSE_NIBBLE;
2032                 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena);
2033                 mkex->keyx_cfg[NIX_INTF_TX] = tx_kex;
2034         }
2035
2036         /* Configure RX interfaces */
2037         for (intf = 0; intf < hw->npc_intfs; intf++) {
2038                 if (is_npc_intf_tx(intf))
2039                         continue;
2040
2041                 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */
2042                 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
2043                             rx_kex);
2044
2045                 /* If MCAM lookup doesn't result in a match, drop the received
2046                  * packet. And map this action to a counter to count dropped
2047                  * packets.
2048                  */
2049                 rvu_write64(rvu, blkaddr,
2050                             NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP);
2051
2052                 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9]
2053                  * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0]
2054                  */
2055                 rvu_write64(rvu, blkaddr,
2056                             NPC_AF_INTFX_MISS_STAT_ACT(intf),
2057                             ((mcam->rx_miss_act_cntr >> 9) << 12) |
2058                             hw->npc_stat_ena | mcam->rx_miss_act_cntr);
2059         }
2060
2061         /* Configure TX interfaces */
2062         for (intf = 0; intf < hw->npc_intfs; intf++) {
2063                 if (is_npc_intf_rx(intf))
2064                         continue;
2065
2066                 /* Extract Ltypes LID_LA to LID_LE */
2067                 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
2068                             tx_kex);
2069
2070                 /* Set TX miss action to UCAST_DEFAULT i.e
2071                  * transmit the packet on NIX LF SQ's default channel.
2072                  */
2073                 rvu_write64(rvu, blkaddr,
2074                             NPC_AF_INTFX_MISS_ACT(intf),
2075                             NIX_TX_ACTIONOP_UCAST_DEFAULT);
2076         }
2077 }
2078
2079 int rvu_npc_init(struct rvu *rvu)
2080 {
2081         struct npc_kpu_profile_adapter *kpu = &rvu->kpu;
2082         struct npc_pkind *pkind = &rvu->hw->pkind;
2083         struct npc_mcam *mcam = &rvu->hw->mcam;
2084         int blkaddr, entry, bank, err;
2085
2086         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2087         if (blkaddr < 0) {
2088                 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
2089                 return -ENODEV;
2090         }
2091
2092         rvu_npc_hw_init(rvu, blkaddr);
2093
2094         /* First disable all MCAM entries, to stop traffic towards NIXLFs */
2095         for (bank = 0; bank < mcam->banks; bank++) {
2096                 for (entry = 0; entry < mcam->banksize; entry++)
2097                         rvu_write64(rvu, blkaddr,
2098                                     NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0);
2099         }
2100
2101         err = rvu_alloc_bitmap(&pkind->rsrc);
2102         if (err)
2103                 return err;
2104         /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0',
2105          * no need to configure PKIND for all LBKs separately.
2106          */
2107         rvu_alloc_rsrc(&pkind->rsrc);
2108
2109         /* Allocate mem for pkind to PF and channel mapping info */
2110         pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max,
2111                                          sizeof(u32), GFP_KERNEL);
2112         if (!pkind->pfchan_map)
2113                 return -ENOMEM;
2114
2115         /* Configure KPU profile */
2116         npc_parser_profile_init(rvu, blkaddr);
2117
2118         /* Config Outer L2, IPv4's NPC layer info */
2119         rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2,
2120                     (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) |
2121                     kpu->lt_def->pck_ol2.ltype_mask);
2122         rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4,
2123                     (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) |
2124                     kpu->lt_def->pck_oip4.ltype_mask);
2125
2126         /* Config Inner IPV4 NPC layer info */
2127         rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4,
2128                     (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) |
2129                     kpu->lt_def->pck_iip4.ltype_mask);
2130
2131         /* Enable below for Rx pkts.
2132          * - Outer IPv4 header checksum validation.
2133          * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B].
2134          * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M].
2135          * - Inner IPv4 header checksum validation.
2136          * - Set non zero checksum error code value
2137          */
2138         rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG,
2139                     rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) |
2140                     ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) |
2141                     BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1));
2142
2143         rvu_npc_setup_interfaces(rvu, blkaddr);
2144
2145         npc_config_secret_key(rvu, blkaddr);
2146         /* Configure MKEX profile */
2147         npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name);
2148
2149         err = npc_mcam_rsrcs_init(rvu, blkaddr);
2150         if (err)
2151                 return err;
2152
2153         err = npc_flow_steering_init(rvu, blkaddr);
2154         if (err) {
2155                 dev_err(rvu->dev,
2156                         "Incorrect mkex profile loaded using default mkex\n");
2157                 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name);
2158         }
2159
2160         return 0;
2161 }
2162
2163 void rvu_npc_freemem(struct rvu *rvu)
2164 {
2165         struct npc_pkind *pkind = &rvu->hw->pkind;
2166         struct npc_mcam *mcam = &rvu->hw->mcam;
2167
2168         kfree(pkind->rsrc.bmap);
2169         npc_mcam_rsrcs_deinit(rvu);
2170         if (rvu->kpu_prfl_addr)
2171                 iounmap(rvu->kpu_prfl_addr);
2172         else
2173                 kfree(rvu->kpu_fwdata);
2174         mutex_destroy(&mcam->lock);
2175 }
2176
2177 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc,
2178                                        int blkaddr, int *alloc_cnt,
2179                                        int *enable_cnt)
2180 {
2181         struct npc_mcam *mcam = &rvu->hw->mcam;
2182         int entry;
2183
2184         *alloc_cnt = 0;
2185         *enable_cnt = 0;
2186
2187         for (entry = 0; entry < mcam->bmap_entries; entry++) {
2188                 if (mcam->entry2pfvf_map[entry] == pcifunc) {
2189                         (*alloc_cnt)++;
2190                         if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry))
2191                                 (*enable_cnt)++;
2192                 }
2193         }
2194 }
2195
2196 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc,
2197                                          int blkaddr, int *alloc_cnt,
2198                                          int *enable_cnt)
2199 {
2200         struct npc_mcam *mcam = &rvu->hw->mcam;
2201         int cntr;
2202
2203         *alloc_cnt = 0;
2204         *enable_cnt = 0;
2205
2206         for (cntr = 0; cntr < mcam->counters.max; cntr++) {
2207                 if (mcam->cntr2pfvf_map[cntr] == pcifunc) {
2208                         (*alloc_cnt)++;
2209                         if (mcam->cntr_refcnt[cntr])
2210                                 (*enable_cnt)++;
2211                 }
2212         }
2213 }
2214
2215 static int npc_mcam_verify_entry(struct npc_mcam *mcam,
2216                                  u16 pcifunc, int entry)
2217 {
2218         /* verify AF installed entries */
2219         if (is_pffunc_af(pcifunc))
2220                 return 0;
2221         /* Verify if entry is valid and if it is indeed
2222          * allocated to the requesting PFFUNC.
2223          */
2224         if (entry >= mcam->bmap_entries)
2225                 return NPC_MCAM_INVALID_REQ;
2226
2227         if (pcifunc != mcam->entry2pfvf_map[entry])
2228                 return NPC_MCAM_PERM_DENIED;
2229
2230         return 0;
2231 }
2232
2233 static int npc_mcam_verify_counter(struct npc_mcam *mcam,
2234                                    u16 pcifunc, int cntr)
2235 {
2236         /* Verify if counter is valid and if it is indeed
2237          * allocated to the requesting PFFUNC.
2238          */
2239         if (cntr >= mcam->counters.max)
2240                 return NPC_MCAM_INVALID_REQ;
2241
2242         if (pcifunc != mcam->cntr2pfvf_map[cntr])
2243                 return NPC_MCAM_PERM_DENIED;
2244
2245         return 0;
2246 }
2247
2248 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam,
2249                                         int blkaddr, u16 entry, u16 cntr)
2250 {
2251         u16 index = entry & (mcam->banksize - 1);
2252         u32 bank = npc_get_bank(mcam, entry);
2253         struct rvu_hwinfo *hw = rvu->hw;
2254
2255         /* Set mapping and increment counter's refcnt */
2256         mcam->entry2cntr_map[entry] = cntr;
2257         mcam->cntr_refcnt[cntr]++;
2258         /* Enable stats */
2259         rvu_write64(rvu, blkaddr,
2260                     NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank),
2261                     ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr);
2262 }
2263
2264 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu,
2265                                           struct npc_mcam *mcam,
2266                                           int blkaddr, u16 entry, u16 cntr)
2267 {
2268         u16 index = entry & (mcam->banksize - 1);
2269         u32 bank = npc_get_bank(mcam, entry);
2270
2271         /* Remove mapping and reduce counter's refcnt */
2272         mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP;
2273         mcam->cntr_refcnt[cntr]--;
2274         /* Disable stats */
2275         rvu_write64(rvu, blkaddr,
2276                     NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00);
2277 }
2278
2279 /* Sets MCAM entry in bitmap as used. Update
2280  * reverse bitmap too. Should be called with
2281  * 'mcam->lock' held.
2282  */
2283 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index)
2284 {
2285         u16 entry, rentry;
2286
2287         entry = index;
2288         rentry = mcam->bmap_entries - index - 1;
2289
2290         __set_bit(entry, mcam->bmap);
2291         __set_bit(rentry, mcam->bmap_reverse);
2292         mcam->bmap_fcnt--;
2293 }
2294
2295 /* Sets MCAM entry in bitmap as free. Update
2296  * reverse bitmap too. Should be called with
2297  * 'mcam->lock' held.
2298  */
2299 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index)
2300 {
2301         u16 entry, rentry;
2302
2303         entry = index;
2304         rentry = mcam->bmap_entries - index - 1;
2305
2306         __clear_bit(entry, mcam->bmap);
2307         __clear_bit(rentry, mcam->bmap_reverse);
2308         mcam->bmap_fcnt++;
2309 }
2310
2311 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,
2312                                       int blkaddr, u16 pcifunc)
2313 {
2314         u16 index, cntr;
2315
2316         /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */
2317         for (index = 0; index < mcam->bmap_entries; index++) {
2318                 if (mcam->entry2pfvf_map[index] == pcifunc) {
2319                         mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP;
2320                         /* Free the entry in bitmap */
2321                         npc_mcam_clear_bit(mcam, index);
2322                         /* Disable the entry */
2323                         npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false);
2324
2325                         /* Update entry2counter mapping */
2326                         cntr = mcam->entry2cntr_map[index];
2327                         if (cntr != NPC_MCAM_INVALID_MAP)
2328                                 npc_unmap_mcam_entry_and_cntr(rvu, mcam,
2329                                                               blkaddr, index,
2330                                                               cntr);
2331                         mcam->entry2target_pffunc[index] = 0x0;
2332                 }
2333         }
2334 }
2335
2336 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam,
2337                                        u16 pcifunc)
2338 {
2339         u16 cntr;
2340
2341         /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */
2342         for (cntr = 0; cntr < mcam->counters.max; cntr++) {
2343                 if (mcam->cntr2pfvf_map[cntr] == pcifunc) {
2344                         mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP;
2345                         mcam->cntr_refcnt[cntr] = 0;
2346                         rvu_free_rsrc(&mcam->counters, cntr);
2347                         /* This API is expected to be called after freeing
2348                          * MCAM entries, which inturn will remove
2349                          * 'entry to counter' mapping.
2350                          * No need to do it again.
2351                          */
2352                 }
2353         }
2354 }
2355
2356 /* Find area of contiguous free entries of size 'nr'.
2357  * If not found return max contiguous free entries available.
2358  */
2359 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start,
2360                                    u16 nr, u16 *max_area)
2361 {
2362         u16 max_area_start = 0;
2363         u16 index, next, end;
2364
2365         *max_area = 0;
2366
2367 again:
2368         index = find_next_zero_bit(map, size, start);
2369         if (index >= size)
2370                 return max_area_start;
2371
2372         end = ((index + nr) >= size) ? size : index + nr;
2373         next = find_next_bit(map, end, index);
2374         if (*max_area < (next - index)) {
2375                 *max_area = next - index;
2376                 max_area_start = index;
2377         }
2378
2379         if (next < end) {
2380                 start = next + 1;
2381                 goto again;
2382         }
2383
2384         return max_area_start;
2385 }
2386
2387 /* Find number of free MCAM entries available
2388  * within range i.e in between 'start' and 'end'.
2389  */
2390 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end)
2391 {
2392         u16 index, next;
2393         u16 fcnt = 0;
2394
2395 again:
2396         if (start >= end)
2397                 return fcnt;
2398
2399         index = find_next_zero_bit(map, end, start);
2400         if (index >= end)
2401                 return fcnt;
2402
2403         next = find_next_bit(map, end, index);
2404         if (next <= end) {
2405                 fcnt += next - index;
2406                 start = next + 1;
2407                 goto again;
2408         }
2409
2410         fcnt += end - index;
2411         return fcnt;
2412 }
2413
2414 static void
2415 npc_get_mcam_search_range_priority(struct npc_mcam *mcam,
2416                                    struct npc_mcam_alloc_entry_req *req,
2417                                    u16 *start, u16 *end, bool *reverse)
2418 {
2419         u16 fcnt;
2420
2421         if (req->priority == NPC_MCAM_HIGHER_PRIO)
2422                 goto hprio;
2423
2424         /* For a low priority entry allocation
2425          * - If reference entry is not in hprio zone then
2426          *      search range: ref_entry to end.
2427          * - If reference entry is in hprio zone and if
2428          *   request can be accomodated in non-hprio zone then
2429          *      search range: 'start of middle zone' to 'end'
2430          * - else search in reverse, so that less number of hprio
2431          *   zone entries are allocated.
2432          */
2433
2434         *reverse = false;
2435         *start = req->ref_entry + 1;
2436         *end = mcam->bmap_entries;
2437
2438         if (req->ref_entry >= mcam->hprio_end)
2439                 return;
2440
2441         fcnt = npc_mcam_get_free_count(mcam->bmap,
2442                                        mcam->hprio_end, mcam->bmap_entries);
2443         if (fcnt > req->count)
2444                 *start = mcam->hprio_end;
2445         else
2446                 *reverse = true;
2447         return;
2448
2449 hprio:
2450         /* For a high priority entry allocation, search is always
2451          * in reverse to preserve hprio zone entries.
2452          * - If reference entry is not in lprio zone then
2453          *      search range: 0 to ref_entry.
2454          * - If reference entry is in lprio zone and if
2455          *   request can be accomodated in middle zone then
2456          *      search range: 'hprio_end' to 'lprio_start'
2457          */
2458
2459         *reverse = true;
2460         *start = 0;
2461         *end = req->ref_entry;
2462
2463         if (req->ref_entry <= mcam->lprio_start)
2464                 return;
2465
2466         fcnt = npc_mcam_get_free_count(mcam->bmap,
2467                                        mcam->hprio_end, mcam->lprio_start);
2468         if (fcnt < req->count)
2469                 return;
2470         *start = mcam->hprio_end;
2471         *end = mcam->lprio_start;
2472 }
2473
2474 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,
2475                                   struct npc_mcam_alloc_entry_req *req,
2476                                   struct npc_mcam_alloc_entry_rsp *rsp)
2477 {
2478         u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES];
2479         u16 fcnt, hp_fcnt, lp_fcnt;
2480         u16 start, end, index;
2481         int entry, next_start;
2482         bool reverse = false;
2483         unsigned long *bmap;
2484         u16 max_contig;
2485
2486         mutex_lock(&mcam->lock);
2487
2488         /* Check if there are any free entries */
2489         if (!mcam->bmap_fcnt) {
2490                 mutex_unlock(&mcam->lock);
2491                 return NPC_MCAM_ALLOC_FAILED;
2492         }
2493
2494         /* MCAM entries are divided into high priority, middle and
2495          * low priority zones. Idea is to not allocate top and lower
2496          * most entries as much as possible, this is to increase
2497          * probability of honouring priority allocation requests.
2498          *
2499          * Two bitmaps are used for mcam entry management,
2500          * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'.
2501          * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'.
2502          *
2503          * Reverse bitmap is used to allocate entries
2504          * - when a higher priority entry is requested
2505          * - when available free entries are less.
2506          * Lower priority ones out of avaialble free entries are always
2507          * chosen when 'high vs low' question arises.
2508          *
2509          * For a VF base MCAM match rule is set by its PF. And all the
2510          * further MCAM rules installed by VF on its own are
2511          * concatenated with the base rule set by its PF. Hence PF entries
2512          * should be at lower priority compared to VF entries. Otherwise
2513          * base rule is hit always and rules installed by VF will be of
2514          * no use. Hence if the request is from PF then allocate low
2515          * priority entries.
2516          */
2517         if (!(pcifunc & RVU_PFVF_FUNC_MASK))
2518                 goto lprio_alloc;
2519
2520         /* Get the search range for priority allocation request */
2521         if (req->priority) {
2522                 npc_get_mcam_search_range_priority(mcam, req,
2523                                                    &start, &end, &reverse);
2524                 goto alloc;
2525         }
2526
2527         /* Find out the search range for non-priority allocation request
2528          *
2529          * Get MCAM free entry count in middle zone.
2530          */
2531         lp_fcnt = npc_mcam_get_free_count(mcam->bmap,
2532                                           mcam->lprio_start,
2533                                           mcam->bmap_entries);
2534         hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end);
2535         fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt;
2536
2537         /* Check if request can be accomodated in the middle zone */
2538         if (fcnt > req->count) {
2539                 start = mcam->hprio_end;
2540                 end = mcam->lprio_start;
2541         } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) {
2542                 /* Expand search zone from half of hprio zone to
2543                  * half of lprio zone.
2544                  */
2545                 start = mcam->hprio_end / 2;
2546                 end = mcam->bmap_entries - (mcam->lprio_count / 2);
2547                 reverse = true;
2548         } else {
2549                 /* Not enough free entries, search all entries in reverse,
2550                  * so that low priority ones will get used up.
2551                  */
2552 lprio_alloc:
2553                 reverse = true;
2554                 start = 0;
2555                 end = mcam->bmap_entries;
2556                 /* Ensure PF requests are always at bottom and if PF requests
2557                  * for higher/lower priority entry wrt reference entry then
2558                  * honour that criteria and start search for entries from bottom
2559                  * and not in mid zone.
2560                  */
2561                 if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&
2562                     req->priority == NPC_MCAM_HIGHER_PRIO)
2563                         end = req->ref_entry;
2564
2565                 if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&
2566                     req->priority == NPC_MCAM_LOWER_PRIO)
2567                         start = req->ref_entry;
2568         }
2569
2570 alloc:
2571         if (reverse) {
2572                 bmap = mcam->bmap_reverse;
2573                 start = mcam->bmap_entries - start;
2574                 end = mcam->bmap_entries - end;
2575                 swap(start, end);
2576         } else {
2577                 bmap = mcam->bmap;
2578         }
2579
2580         if (req->contig) {
2581                 /* Allocate requested number of contiguous entries, if
2582                  * unsuccessful find max contiguous entries available.
2583                  */
2584                 index = npc_mcam_find_zero_area(bmap, end, start,
2585                                                 req->count, &max_contig);
2586                 rsp->count = max_contig;
2587                 if (reverse)
2588                         rsp->entry = mcam->bmap_entries - index - max_contig;
2589                 else
2590                         rsp->entry = index;
2591         } else {
2592                 /* Allocate requested number of non-contiguous entries,
2593                  * if unsuccessful allocate as many as possible.
2594                  */
2595                 rsp->count = 0;
2596                 next_start = start;
2597                 for (entry = 0; entry < req->count; entry++) {
2598                         index = find_next_zero_bit(bmap, end, next_start);
2599                         if (index >= end)
2600                                 break;
2601
2602                         next_start = start + (index - start) + 1;
2603
2604                         /* Save the entry's index */
2605                         if (reverse)
2606                                 index = mcam->bmap_entries - index - 1;
2607                         entry_list[entry] = index;
2608                         rsp->count++;
2609                 }
2610         }
2611
2612         /* If allocating requested no of entries is unsucessful,
2613          * expand the search range to full bitmap length and retry.
2614          */
2615         if (!req->priority && (rsp->count < req->count) &&
2616             ((end - start) != mcam->bmap_entries)) {
2617                 reverse = true;
2618                 start = 0;
2619                 end = mcam->bmap_entries;
2620                 goto alloc;
2621         }
2622
2623         /* For priority entry allocation requests, if allocation is
2624          * failed then expand search to max possible range and retry.
2625          */
2626         if (req->priority && rsp->count < req->count) {
2627                 if (req->priority == NPC_MCAM_LOWER_PRIO &&
2628                     (start != (req->ref_entry + 1))) {
2629                         start = req->ref_entry + 1;
2630                         end = mcam->bmap_entries;
2631                         reverse = false;
2632                         goto alloc;
2633                 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) &&
2634                            ((end - start) != req->ref_entry)) {
2635                         start = 0;
2636                         end = req->ref_entry;
2637                         reverse = true;
2638                         goto alloc;
2639                 }
2640         }
2641
2642         /* Copy MCAM entry indices into mbox response entry_list.
2643          * Requester always expects indices in ascending order, so
2644          * reverse the list if reverse bitmap is used for allocation.
2645          */
2646         if (!req->contig && rsp->count) {
2647                 index = 0;
2648                 for (entry = rsp->count - 1; entry >= 0; entry--) {
2649                         if (reverse)
2650                                 rsp->entry_list[index++] = entry_list[entry];
2651                         else
2652                                 rsp->entry_list[entry] = entry_list[entry];
2653                 }
2654         }
2655
2656         /* Mark the allocated entries as used and set nixlf mapping */
2657         for (entry = 0; entry < rsp->count; entry++) {
2658                 index = req->contig ?
2659                         (rsp->entry + entry) : rsp->entry_list[entry];
2660                 npc_mcam_set_bit(mcam, index);
2661                 mcam->entry2pfvf_map[index] = pcifunc;
2662                 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP;
2663         }
2664
2665         /* Update available free count in mbox response */
2666         rsp->free_count = mcam->bmap_fcnt;
2667
2668         mutex_unlock(&mcam->lock);
2669         return 0;
2670 }
2671
2672 /* Marks bitmaps to reserved the mcam slot */
2673 void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx)
2674 {
2675         struct npc_mcam *mcam = &rvu->hw->mcam;
2676
2677         npc_mcam_set_bit(mcam, entry_idx);
2678 }
2679
2680 int npc_config_cntr_default_entries(struct rvu *rvu, bool enable)
2681 {
2682         struct npc_mcam *mcam = &rvu->hw->mcam;
2683         struct npc_install_flow_rsp rsp;
2684         struct rvu_npc_mcam_rule *rule;
2685         int blkaddr;
2686
2687         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2688         if (blkaddr < 0)
2689                 return -EINVAL;
2690
2691         mutex_lock(&mcam->lock);
2692         list_for_each_entry(rule, &mcam->mcam_rules, list) {
2693                 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, rule->entry))
2694                         continue;
2695                 if (!rule->default_rule)
2696                         continue;
2697                 if (enable && !rule->has_cntr) { /* Alloc and map new counter */
2698                         __rvu_mcam_add_counter_to_rule(rvu, rule->owner,
2699                                                        rule, &rsp);
2700                         if (rsp.counter < 0) {
2701                                 dev_err(rvu->dev,
2702                                         "%s: Failed to allocate cntr for default rule (err=%d)\n",
2703                                         __func__, rsp.counter);
2704                                 break;
2705                         }
2706                         npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr,
2707                                                     rule->entry, rsp.counter);
2708                         /* Reset counter before use */
2709                         rvu_write64(rvu, blkaddr,
2710                                     NPC_AF_MATCH_STATX(rule->cntr), 0x0);
2711                 }
2712
2713                 /* Free and unmap counter */
2714                 if (!enable && rule->has_cntr)
2715                         __rvu_mcam_remove_counter_from_rule(rvu, rule->owner,
2716                                                             rule);
2717         }
2718         mutex_unlock(&mcam->lock);
2719
2720         return 0;
2721 }
2722
2723 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu,
2724                                           struct npc_mcam_alloc_entry_req *req,
2725                                           struct npc_mcam_alloc_entry_rsp *rsp)
2726 {
2727         struct npc_mcam *mcam = &rvu->hw->mcam;
2728         u16 pcifunc = req->hdr.pcifunc;
2729         int blkaddr;
2730
2731         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2732         if (blkaddr < 0)
2733                 return NPC_MCAM_INVALID_REQ;
2734
2735         rsp->entry = NPC_MCAM_ENTRY_INVALID;
2736         rsp->free_count = 0;
2737
2738         /* Check if ref_entry is greater that the range
2739          * then set it to max value.
2740          */
2741         if (req->ref_entry > mcam->bmap_entries)
2742                 req->ref_entry = mcam->bmap_entries;
2743
2744         /* ref_entry can't be '0' if requested priority is high.
2745          * Can't be last entry if requested priority is low.
2746          */
2747         if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) ||
2748             ((req->ref_entry == mcam->bmap_entries) &&
2749              req->priority == NPC_MCAM_LOWER_PRIO))
2750                 return NPC_MCAM_INVALID_REQ;
2751
2752         /* Since list of allocated indices needs to be sent to requester,
2753          * max number of non-contiguous entries per mbox msg is limited.
2754          */
2755         if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) {
2756                 dev_err(rvu->dev,
2757                         "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n",
2758                         __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES);
2759                 return NPC_MCAM_INVALID_REQ;
2760         }
2761
2762         /* Alloc request from PFFUNC with no NIXLF attached should be denied */
2763         if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))
2764                 return NPC_MCAM_ALLOC_DENIED;
2765
2766         return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp);
2767 }
2768
2769 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu,
2770                                          struct npc_mcam_free_entry_req *req,
2771                                          struct msg_rsp *rsp)
2772 {
2773         struct npc_mcam *mcam = &rvu->hw->mcam;
2774         u16 pcifunc = req->hdr.pcifunc;
2775         int blkaddr, rc = 0;
2776         u16 cntr;
2777
2778         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2779         if (blkaddr < 0)
2780                 return NPC_MCAM_INVALID_REQ;
2781
2782         /* Free request from PFFUNC with no NIXLF attached, ignore */
2783         if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))
2784                 return NPC_MCAM_INVALID_REQ;
2785
2786         mutex_lock(&mcam->lock);
2787
2788         if (req->all)
2789                 goto free_all;
2790
2791         rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
2792         if (rc)
2793                 goto exit;
2794
2795         mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP;
2796         mcam->entry2target_pffunc[req->entry] = 0x0;
2797         npc_mcam_clear_bit(mcam, req->entry);
2798         npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false);
2799
2800         /* Update entry2counter mapping */
2801         cntr = mcam->entry2cntr_map[req->entry];
2802         if (cntr != NPC_MCAM_INVALID_MAP)
2803                 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
2804                                               req->entry, cntr);
2805
2806         goto exit;
2807
2808 free_all:
2809         /* Free up all entries allocated to requesting PFFUNC */
2810         npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc);
2811 exit:
2812         mutex_unlock(&mcam->lock);
2813         return rc;
2814 }
2815
2816 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu,
2817                                          struct npc_mcam_read_entry_req *req,
2818                                          struct npc_mcam_read_entry_rsp *rsp)
2819 {
2820         struct npc_mcam *mcam = &rvu->hw->mcam;
2821         u16 pcifunc = req->hdr.pcifunc;
2822         int blkaddr, rc;
2823
2824         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2825         if (blkaddr < 0)
2826                 return NPC_MCAM_INVALID_REQ;
2827
2828         mutex_lock(&mcam->lock);
2829         rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
2830         if (!rc) {
2831                 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry,
2832                                     &rsp->entry_data,
2833                                     &rsp->intf, &rsp->enable);
2834         }
2835
2836         mutex_unlock(&mcam->lock);
2837         return rc;
2838 }
2839
2840 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu,
2841                                           struct npc_mcam_write_entry_req *req,
2842                                           struct msg_rsp *rsp)
2843 {
2844         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
2845         struct npc_mcam *mcam = &rvu->hw->mcam;
2846         u16 pcifunc = req->hdr.pcifunc;
2847         int blkaddr, rc;
2848         u8 nix_intf;
2849
2850         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2851         if (blkaddr < 0)
2852                 return NPC_MCAM_INVALID_REQ;
2853
2854         mutex_lock(&mcam->lock);
2855         rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
2856         if (rc)
2857                 goto exit;
2858
2859         if (req->set_cntr &&
2860             npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) {
2861                 rc = NPC_MCAM_INVALID_REQ;
2862                 goto exit;
2863         }
2864
2865         if (!is_npc_interface_valid(rvu, req->intf)) {
2866                 rc = NPC_MCAM_INVALID_REQ;
2867                 goto exit;
2868         }
2869
2870         if (is_npc_intf_tx(req->intf))
2871                 nix_intf = pfvf->nix_tx_intf;
2872         else
2873                 nix_intf = pfvf->nix_rx_intf;
2874
2875         /* For AF installed rules, the nix_intf should be set to target NIX */
2876         if (is_pffunc_af(req->hdr.pcifunc))
2877                 nix_intf = req->intf;
2878
2879         npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf,
2880                               &req->entry_data, req->enable_entry);
2881
2882         if (req->set_cntr)
2883                 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr,
2884                                             req->entry, req->cntr);
2885
2886         rc = 0;
2887 exit:
2888         mutex_unlock(&mcam->lock);
2889         return rc;
2890 }
2891
2892 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu,
2893                                         struct npc_mcam_ena_dis_entry_req *req,
2894                                         struct msg_rsp *rsp)
2895 {
2896         struct npc_mcam *mcam = &rvu->hw->mcam;
2897         u16 pcifunc = req->hdr.pcifunc;
2898         int blkaddr, rc;
2899
2900         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2901         if (blkaddr < 0)
2902                 return NPC_MCAM_INVALID_REQ;
2903
2904         mutex_lock(&mcam->lock);
2905         rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
2906         mutex_unlock(&mcam->lock);
2907         if (rc)
2908                 return rc;
2909
2910         npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true);
2911
2912         return 0;
2913 }
2914
2915 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu,
2916                                         struct npc_mcam_ena_dis_entry_req *req,
2917                                         struct msg_rsp *rsp)
2918 {
2919         struct npc_mcam *mcam = &rvu->hw->mcam;
2920         u16 pcifunc = req->hdr.pcifunc;
2921         int blkaddr, rc;
2922
2923         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2924         if (blkaddr < 0)
2925                 return NPC_MCAM_INVALID_REQ;
2926
2927         mutex_lock(&mcam->lock);
2928         rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
2929         mutex_unlock(&mcam->lock);
2930         if (rc)
2931                 return rc;
2932
2933         npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false);
2934
2935         return 0;
2936 }
2937
2938 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu,
2939                                           struct npc_mcam_shift_entry_req *req,
2940                                           struct npc_mcam_shift_entry_rsp *rsp)
2941 {
2942         struct npc_mcam *mcam = &rvu->hw->mcam;
2943         u16 pcifunc = req->hdr.pcifunc;
2944         u16 old_entry, new_entry;
2945         int blkaddr, rc = 0;
2946         u16 index, cntr;
2947
2948         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
2949         if (blkaddr < 0)
2950                 return NPC_MCAM_INVALID_REQ;
2951
2952         if (req->shift_count > NPC_MCAM_MAX_SHIFTS)
2953                 return NPC_MCAM_INVALID_REQ;
2954
2955         mutex_lock(&mcam->lock);
2956         for (index = 0; index < req->shift_count; index++) {
2957                 old_entry = req->curr_entry[index];
2958                 new_entry = req->new_entry[index];
2959
2960                 /* Check if both old and new entries are valid and
2961                  * does belong to this PFFUNC or not.
2962                  */
2963                 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry);
2964                 if (rc)
2965                         break;
2966
2967                 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry);
2968                 if (rc)
2969                         break;
2970
2971                 /* new_entry should not have a counter mapped */
2972                 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) {
2973                         rc = NPC_MCAM_PERM_DENIED;
2974                         break;
2975                 }
2976
2977                 /* Disable the new_entry */
2978                 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false);
2979
2980                 /* Copy rule from old entry to new entry */
2981                 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry);
2982
2983                 /* Copy counter mapping, if any */
2984                 cntr = mcam->entry2cntr_map[old_entry];
2985                 if (cntr != NPC_MCAM_INVALID_MAP) {
2986                         npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
2987                                                       old_entry, cntr);
2988                         npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr,
2989                                                     new_entry, cntr);
2990                 }
2991
2992                 /* Enable new_entry and disable old_entry */
2993                 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true);
2994                 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false);
2995         }
2996
2997         /* If shift has failed then report the failed index */
2998         if (index != req->shift_count) {
2999                 rc = NPC_MCAM_PERM_DENIED;
3000                 rsp->failed_entry_idx = index;
3001         }
3002
3003         mutex_unlock(&mcam->lock);
3004         return rc;
3005 }
3006
3007 static int __npc_mcam_alloc_counter(struct rvu *rvu,
3008                                     struct npc_mcam_alloc_counter_req *req,
3009                                     struct npc_mcam_alloc_counter_rsp *rsp)
3010 {
3011         struct npc_mcam *mcam = &rvu->hw->mcam;
3012         u16 pcifunc = req->hdr.pcifunc;
3013         u16 max_contig, cntr;
3014         int blkaddr, index;
3015
3016         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3017         if (blkaddr < 0)
3018                 return NPC_MCAM_INVALID_REQ;
3019
3020         /* If the request is from a PFFUNC with no NIXLF attached, ignore */
3021         if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))
3022                 return NPC_MCAM_INVALID_REQ;
3023
3024         /* Since list of allocated counter IDs needs to be sent to requester,
3025          * max number of non-contiguous counters per mbox msg is limited.
3026          */
3027         if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS)
3028                 return NPC_MCAM_INVALID_REQ;
3029
3030
3031         /* Check if unused counters are available or not */
3032         if (!rvu_rsrc_free_count(&mcam->counters)) {
3033                 return NPC_MCAM_ALLOC_FAILED;
3034         }
3035
3036         rsp->count = 0;
3037
3038         if (req->contig) {
3039                 /* Allocate requested number of contiguous counters, if
3040                  * unsuccessful find max contiguous entries available.
3041                  */
3042                 index = npc_mcam_find_zero_area(mcam->counters.bmap,
3043                                                 mcam->counters.max, 0,
3044                                                 req->count, &max_contig);
3045                 rsp->count = max_contig;
3046                 rsp->cntr = index;
3047                 for (cntr = index; cntr < (index + max_contig); cntr++) {
3048                         __set_bit(cntr, mcam->counters.bmap);
3049                         mcam->cntr2pfvf_map[cntr] = pcifunc;
3050                 }
3051         } else {
3052                 /* Allocate requested number of non-contiguous counters,
3053                  * if unsuccessful allocate as many as possible.
3054                  */
3055                 for (cntr = 0; cntr < req->count; cntr++) {
3056                         index = rvu_alloc_rsrc(&mcam->counters);
3057                         if (index < 0)
3058                                 break;
3059                         rsp->cntr_list[cntr] = index;
3060                         rsp->count++;
3061                         mcam->cntr2pfvf_map[index] = pcifunc;
3062                 }
3063         }
3064
3065         return 0;
3066 }
3067
3068 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu,
3069                         struct npc_mcam_alloc_counter_req *req,
3070                         struct npc_mcam_alloc_counter_rsp *rsp)
3071 {
3072         struct npc_mcam *mcam = &rvu->hw->mcam;
3073         int err;
3074
3075         mutex_lock(&mcam->lock);
3076
3077         err = __npc_mcam_alloc_counter(rvu, req, rsp);
3078
3079         mutex_unlock(&mcam->lock);
3080         return err;
3081 }
3082
3083 static int __npc_mcam_free_counter(struct rvu *rvu,
3084                                    struct npc_mcam_oper_counter_req *req,
3085                                    struct msg_rsp *rsp)
3086 {
3087         struct npc_mcam *mcam = &rvu->hw->mcam;
3088         u16 index, entry = 0;
3089         int blkaddr, err;
3090
3091         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3092         if (blkaddr < 0)
3093                 return NPC_MCAM_INVALID_REQ;
3094
3095         err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
3096         if (err) {
3097                 return err;
3098         }
3099
3100         /* Mark counter as free/unused */
3101         mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP;
3102         rvu_free_rsrc(&mcam->counters, req->cntr);
3103
3104         /* Disable all MCAM entry's stats which are using this counter */
3105         while (entry < mcam->bmap_entries) {
3106                 if (!mcam->cntr_refcnt[req->cntr])
3107                         break;
3108
3109                 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry);
3110                 if (index >= mcam->bmap_entries)
3111                         break;
3112                 entry = index + 1;
3113                 if (mcam->entry2cntr_map[index] != req->cntr)
3114                         continue;
3115
3116                 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
3117                                               index, req->cntr);
3118         }
3119
3120         return 0;
3121 }
3122
3123 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu,
3124                 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp)
3125 {
3126         struct npc_mcam *mcam = &rvu->hw->mcam;
3127         int err;
3128
3129         mutex_lock(&mcam->lock);
3130
3131         err = __npc_mcam_free_counter(rvu, req, rsp);
3132
3133         mutex_unlock(&mcam->lock);
3134
3135         return err;
3136 }
3137
3138 void __rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
3139                                          struct rvu_npc_mcam_rule *rule)
3140 {
3141         struct npc_mcam_oper_counter_req free_req = { 0 };
3142         struct msg_rsp free_rsp;
3143
3144         if (!rule->has_cntr)
3145                 return;
3146
3147         free_req.hdr.pcifunc = pcifunc;
3148         free_req.cntr = rule->cntr;
3149
3150         __npc_mcam_free_counter(rvu, &free_req, &free_rsp);
3151         rule->has_cntr = false;
3152 }
3153
3154 void __rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc,
3155                                     struct rvu_npc_mcam_rule *rule,
3156                                     struct npc_install_flow_rsp *rsp)
3157 {
3158         struct npc_mcam_alloc_counter_req cntr_req = { 0 };
3159         struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
3160         int err;
3161
3162         cntr_req.hdr.pcifunc = pcifunc;
3163         cntr_req.contig = true;
3164         cntr_req.count = 1;
3165
3166         /* we try to allocate a counter to track the stats of this
3167          * rule. If counter could not be allocated then proceed
3168          * without counter because counters are limited than entries.
3169          */
3170         err = __npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp);
3171         if (!err && cntr_rsp.count) {
3172                 rule->cntr = cntr_rsp.cntr;
3173                 rule->has_cntr = true;
3174                 rsp->counter = rule->cntr;
3175         } else {
3176                 rsp->counter = err;
3177         }
3178 }
3179
3180 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu,
3181                 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp)
3182 {
3183         struct npc_mcam *mcam = &rvu->hw->mcam;
3184         u16 index, entry = 0;
3185         int blkaddr, rc;
3186
3187         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3188         if (blkaddr < 0)
3189                 return NPC_MCAM_INVALID_REQ;
3190
3191         mutex_lock(&mcam->lock);
3192         rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
3193         if (rc)
3194                 goto exit;
3195
3196         /* Unmap the MCAM entry and counter */
3197         if (!req->all) {
3198                 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry);
3199                 if (rc)
3200                         goto exit;
3201                 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
3202                                               req->entry, req->cntr);
3203                 goto exit;
3204         }
3205
3206         /* Disable all MCAM entry's stats which are using this counter */
3207         while (entry < mcam->bmap_entries) {
3208                 if (!mcam->cntr_refcnt[req->cntr])
3209                         break;
3210
3211                 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry);
3212                 if (index >= mcam->bmap_entries)
3213                         break;
3214                 entry = index + 1;
3215
3216                 if (mcam->entry2cntr_map[index] != req->cntr)
3217                         continue;
3218
3219                 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,
3220                                               index, req->cntr);
3221         }
3222 exit:
3223         mutex_unlock(&mcam->lock);
3224         return rc;
3225 }
3226
3227 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu,
3228                 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp)
3229 {
3230         struct npc_mcam *mcam = &rvu->hw->mcam;
3231         int blkaddr, err;
3232
3233         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3234         if (blkaddr < 0)
3235                 return NPC_MCAM_INVALID_REQ;
3236
3237         mutex_lock(&mcam->lock);
3238         err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
3239         mutex_unlock(&mcam->lock);
3240         if (err)
3241                 return err;
3242
3243         rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00);
3244
3245         return 0;
3246 }
3247
3248 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu,
3249                         struct npc_mcam_oper_counter_req *req,
3250                         struct npc_mcam_oper_counter_rsp *rsp)
3251 {
3252         struct npc_mcam *mcam = &rvu->hw->mcam;
3253         int blkaddr, err;
3254
3255         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3256         if (blkaddr < 0)
3257                 return NPC_MCAM_INVALID_REQ;
3258
3259         mutex_lock(&mcam->lock);
3260         err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);
3261         mutex_unlock(&mcam->lock);
3262         if (err)
3263                 return err;
3264
3265         rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr));
3266         rsp->stat &= BIT_ULL(48) - 1;
3267
3268         return 0;
3269 }
3270
3271 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu,
3272                           struct npc_mcam_alloc_and_write_entry_req *req,
3273                           struct npc_mcam_alloc_and_write_entry_rsp *rsp)
3274 {
3275         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
3276         struct npc_mcam_alloc_counter_req cntr_req;
3277         struct npc_mcam_alloc_counter_rsp cntr_rsp;
3278         struct npc_mcam_alloc_entry_req entry_req;
3279         struct npc_mcam_alloc_entry_rsp entry_rsp;
3280         struct npc_mcam *mcam = &rvu->hw->mcam;
3281         u16 entry = NPC_MCAM_ENTRY_INVALID;
3282         u16 cntr = NPC_MCAM_ENTRY_INVALID;
3283         int blkaddr, rc;
3284         u8 nix_intf;
3285
3286         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3287         if (blkaddr < 0)
3288                 return NPC_MCAM_INVALID_REQ;
3289
3290         if (!is_npc_interface_valid(rvu, req->intf))
3291                 return NPC_MCAM_INVALID_REQ;
3292
3293         /* Try to allocate a MCAM entry */
3294         entry_req.hdr.pcifunc = req->hdr.pcifunc;
3295         entry_req.contig = true;
3296         entry_req.priority = req->priority;
3297         entry_req.ref_entry = req->ref_entry;
3298         entry_req.count = 1;
3299
3300         rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,
3301                                                    &entry_req, &entry_rsp);
3302         if (rc)
3303                 return rc;
3304
3305         if (!entry_rsp.count)
3306                 return NPC_MCAM_ALLOC_FAILED;
3307
3308         entry = entry_rsp.entry;
3309
3310         if (!req->alloc_cntr)
3311                 goto write_entry;
3312
3313         /* Now allocate counter */
3314         cntr_req.hdr.pcifunc = req->hdr.pcifunc;
3315         cntr_req.contig = true;
3316         cntr_req.count = 1;
3317
3318         rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp);
3319         if (rc) {
3320                 /* Free allocated MCAM entry */
3321                 mutex_lock(&mcam->lock);
3322                 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP;
3323                 npc_mcam_clear_bit(mcam, entry);
3324                 mutex_unlock(&mcam->lock);
3325                 return rc;
3326         }
3327
3328         cntr = cntr_rsp.cntr;
3329
3330 write_entry:
3331         mutex_lock(&mcam->lock);
3332
3333         if (is_npc_intf_tx(req->intf))
3334                 nix_intf = pfvf->nix_tx_intf;
3335         else
3336                 nix_intf = pfvf->nix_rx_intf;
3337
3338         npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf,
3339                               &req->entry_data, req->enable_entry);
3340
3341         if (req->alloc_cntr)
3342                 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr);
3343         mutex_unlock(&mcam->lock);
3344
3345         rsp->entry = entry;
3346         rsp->cntr = cntr;
3347
3348         return 0;
3349 }
3350
3351 #define GET_KEX_CFG(intf) \
3352         rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf))
3353
3354 #define GET_KEX_FLAGS(ld) \
3355         rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld))
3356
3357 #define GET_KEX_LD(intf, lid, lt, ld)   \
3358         rvu_read64(rvu, BLKADDR_NPC,    \
3359                 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld))
3360
3361 #define GET_KEX_LDFLAGS(intf, ld, fl)   \
3362         rvu_read64(rvu, BLKADDR_NPC,    \
3363                 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl))
3364
3365 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req,
3366                                      struct npc_get_kex_cfg_rsp *rsp)
3367 {
3368         int lid, lt, ld, fl;
3369
3370         rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX);
3371         rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX);
3372         for (lid = 0; lid < NPC_MAX_LID; lid++) {
3373                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
3374                         for (ld = 0; ld < NPC_MAX_LD; ld++) {
3375                                 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] =
3376                                         GET_KEX_LD(NIX_INTF_RX, lid, lt, ld);
3377                                 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] =
3378                                         GET_KEX_LD(NIX_INTF_TX, lid, lt, ld);
3379                         }
3380                 }
3381         }
3382         for (ld = 0; ld < NPC_MAX_LD; ld++)
3383                 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld);
3384
3385         for (ld = 0; ld < NPC_MAX_LD; ld++) {
3386                 for (fl = 0; fl < NPC_MAX_LFL; fl++) {
3387                         rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] =
3388                                         GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl);
3389                         rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] =
3390                                         GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl);
3391                 }
3392         }
3393         memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN);
3394         return 0;
3395 }
3396
3397 static int
3398 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind,
3399                              u8 var_len_off, u8 var_len_off_mask, u8 shift_dir)
3400 {
3401         struct npc_kpu_action0 *act0;
3402         u8 shift_count = 0;
3403         int blkaddr;
3404         u64 val;
3405
3406         if (!var_len_off_mask)
3407                 return -EINVAL;
3408
3409         if (var_len_off_mask != 0xff) {
3410                 if (shift_dir)
3411                         shift_count = __ffs(var_len_off_mask);
3412                 else
3413                         shift_count = (8 - __fls(var_len_off_mask));
3414         }
3415         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);
3416         if (blkaddr < 0) {
3417                 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
3418                 return -EINVAL;
3419         }
3420         val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));
3421         act0 = (struct npc_kpu_action0 *)&val;
3422         act0->var_len_shift = shift_count;
3423         act0->var_len_right = shift_dir;
3424         act0->var_len_mask = var_len_off_mask;
3425         act0->var_len_offset = var_len_off;
3426         rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);
3427         return 0;
3428 }
3429
3430 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
3431                            u64 pkind, u8 var_len_off, u8 var_len_off_mask,
3432                            u8 shift_dir)
3433
3434 {
3435         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
3436         int blkaddr, nixlf, rc, intf_mode;
3437         int pf = rvu_get_pf(pcifunc);
3438         u64 rxpkind, txpkind;
3439         u8 cgx_id, lmac_id;
3440
3441         /* use default pkind to disable edsa/higig */
3442         rxpkind = rvu_npc_get_pkind(rvu, pf);
3443         txpkind = NPC_TX_DEF_PKIND;
3444         intf_mode = NPC_INTF_MODE_DEF;
3445
3446         if (mode & OTX2_PRIV_FLAGS_CUSTOM) {
3447                 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) {
3448                         rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind,
3449                                                           var_len_off,
3450                                                           var_len_off_mask,
3451                                                           shift_dir);
3452                         if (rc)
3453                                 return rc;
3454                 }
3455                 rxpkind = pkind;
3456                 txpkind = pkind;
3457         }
3458
3459         if (dir & PKIND_RX) {
3460                 /* rx pkind set req valid only for cgx mapped PFs */
3461                 if (!is_cgx_config_permitted(rvu, pcifunc))
3462                         return 0;
3463                 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);
3464
3465                 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id,
3466                                    rxpkind);
3467                 if (rc)
3468                         return rc;
3469         }
3470
3471         if (dir & PKIND_TX) {
3472                 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */
3473                 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);
3474                 if (rc)
3475                         return rc;
3476
3477                 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf),
3478                             txpkind);
3479         }
3480
3481         pfvf->intf_mode = intf_mode;
3482         return 0;
3483 }
3484
3485 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req,
3486                                    struct msg_rsp *rsp)
3487 {
3488         return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode,
3489                                       req->dir, req->pkind, req->var_len_off,
3490                                       req->var_len_off_mask, req->shift_dir);
3491 }
3492
3493 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu,
3494                                               struct msg_req *req,
3495                                               struct npc_mcam_read_base_rule_rsp *rsp)
3496 {
3497         struct npc_mcam *mcam = &rvu->hw->mcam;
3498         int index, blkaddr, nixlf, rc = 0;
3499         u16 pcifunc = req->hdr.pcifunc;
3500         struct rvu_pfvf *pfvf;
3501         u8 intf, enable;
3502
3503         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3504         if (blkaddr < 0)
3505                 return NPC_MCAM_INVALID_REQ;
3506
3507         /* Return the channel number in case of PF */
3508         if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
3509                 pfvf = rvu_get_pfvf(rvu, pcifunc);
3510                 rsp->entry.kw[0] = pfvf->rx_chan_base;
3511                 rsp->entry.kw_mask[0] = 0xFFFULL;
3512                 goto out;
3513         }
3514
3515         /* Find the pkt steering rule installed by PF to this VF */
3516         mutex_lock(&mcam->lock);
3517         for (index = 0; index < mcam->bmap_entries; index++) {
3518                 if (mcam->entry2target_pffunc[index] == pcifunc)
3519                         goto read_entry;
3520         }
3521
3522         rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
3523         if (rc < 0) {
3524                 mutex_unlock(&mcam->lock);
3525                 goto out;
3526         }
3527         /* Read the default ucast entry if there is no pkt steering rule */
3528         index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,
3529                                          NIXLF_UCAST_ENTRY);
3530 read_entry:
3531         /* Read the mcam entry */
3532         npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf,
3533                             &enable);
3534         mutex_unlock(&mcam->lock);
3535 out:
3536         return rc;
3537 }
3538
3539 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu,
3540                                           struct npc_mcam_get_stats_req *req,
3541                                           struct npc_mcam_get_stats_rsp *rsp)
3542 {
3543         struct npc_mcam *mcam = &rvu->hw->mcam;
3544         u16 index, cntr;
3545         int blkaddr;
3546         u64 regval;
3547         u32 bank;
3548
3549         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3550         if (blkaddr < 0)
3551                 return NPC_MCAM_INVALID_REQ;
3552
3553         mutex_lock(&mcam->lock);
3554
3555         index = req->entry & (mcam->banksize - 1);
3556         bank = npc_get_bank(mcam, req->entry);
3557
3558         /* read MCAM entry STAT_ACT register */
3559         regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank));
3560
3561         if (!(regval & rvu->hw->npc_stat_ena)) {
3562                 rsp->stat_ena = 0;
3563                 mutex_unlock(&mcam->lock);
3564                 return 0;
3565         }
3566
3567         cntr = regval & 0x1FF;
3568
3569         rsp->stat_ena = 1;
3570         rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr));
3571         rsp->stat &= BIT_ULL(48) - 1;
3572
3573         mutex_unlock(&mcam->lock);
3574
3575         return 0;
3576 }
3577
3578 void rvu_npc_clear_ucast_entry(struct rvu *rvu, int pcifunc, int nixlf)
3579 {
3580         struct npc_mcam *mcam = &rvu->hw->mcam;
3581         struct rvu_npc_mcam_rule *rule;
3582         int ucast_idx, blkaddr;
3583
3584         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
3585         if (blkaddr < 0)
3586                 return;
3587
3588         ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,
3589                                              nixlf, NIXLF_UCAST_ENTRY);
3590
3591         npc_enable_mcam_entry(rvu, mcam, blkaddr, ucast_idx, false);
3592
3593         npc_set_mcam_action(rvu, mcam, blkaddr, ucast_idx, 0);
3594
3595         npc_clear_mcam_entry(rvu, mcam, blkaddr, ucast_idx);
3596
3597         mutex_lock(&mcam->lock);
3598         list_for_each_entry(rule, &mcam->mcam_rules, list) {
3599                 if (rule->entry == ucast_idx) {
3600                         list_del(&rule->list);
3601                         kfree(rule);
3602                         break;
3603                 }
3604         }
3605         mutex_unlock(&mcam->lock);
3606 }