drivers/net/ethernet/mediatek/mtk_ppe.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/io.h>
   6 #include <linux/iopoll.h>
   7 #include <linux/etherdevice.h>
   8 #include <linux/platform_device.h>
   9 #include <linux/if_ether.h>
  10 #include <linux/if_vlan.h>
  11 #include <net/dsa.h>
  12 #include "mtk_eth_soc.h"
  13 #include "mtk_ppe.h"
  14 #include "mtk_ppe_regs.h"
  15
  16 static DEFINE_SPINLOCK(ppe_lock);
  17
  18 static const struct rhashtable_params mtk_flow_l2_ht_params = {
  19         .head_offset = offsetof(struct mtk_flow_entry, l2_node),
  20         .key_offset = offsetof(struct mtk_flow_entry, data.bridge),
  21         .key_len = offsetof(struct mtk_foe_bridge, key_end),
  22         .automatic_shrinking = true,
  23 };
  24
  25 static void ppe_w32(struct mtk_ppe *ppe, u32 reg, u32 val)
  26 {
  27         writel(val, ppe->base + reg);
  28 }
  29
  30 static u32 ppe_r32(struct mtk_ppe *ppe, u32 reg)
  31 {
  32         return readl(ppe->base + reg);
  33 }
  34
  35 static u32 ppe_m32(struct mtk_ppe *ppe, u32 reg, u32 mask, u32 set)
  36 {
  37         u32 val;
  38
  39         val = ppe_r32(ppe, reg);
  40         val &= ~mask;
  41         val |= set;
  42         ppe_w32(ppe, reg, val);
  43
  44         return val;
  45 }
  46
  47 static u32 ppe_set(struct mtk_ppe *ppe, u32 reg, u32 val)
  48 {
  49         return ppe_m32(ppe, reg, 0, val);
  50 }
  51
  52 static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
  53 {
  54         return ppe_m32(ppe, reg, val, 0);
  55 }
  56
  57 static u32 mtk_eth_timestamp(struct mtk_eth *eth)
  58 {
  59         return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth);
  60 }
  61
  62 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
  63 {
  64         int ret;
  65         u32 val;
  66
  67         ret = readl_poll_timeout(ppe->base + MTK_PPE_GLO_CFG, val,
  68                                  !(val & MTK_PPE_GLO_CFG_BUSY),
  69                                  20, MTK_PPE_WAIT_TIMEOUT_US);
  70
  71         if (ret)
  72                 dev_err(ppe->dev, "PPE table busy");
  73
  74         return ret;
  75 }
  76
  77 static void mtk_ppe_cache_clear(struct mtk_ppe *ppe)
  78 {
  79         ppe_set(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
  80         ppe_clear(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_CLEAR);
  81 }
  82
  83 static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
  84 {
  85         mtk_ppe_cache_clear(ppe);
  86
  87         ppe_m32(ppe, MTK_PPE_CACHE_CTL, MTK_PPE_CACHE_CTL_EN,
  88                 enable * MTK_PPE_CACHE_CTL_EN);
  89 }
  90
  91 static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e)
  92 {
  93         u32 hv1, hv2, hv3;
  94         u32 hash;
  95
  96         switch (mtk_get_ib1_pkt_type(eth, e->ib1)) {
  97                 case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
  98                 case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
  99                         hv1 = e->ipv4.orig.ports;
 100                         hv2 = e->ipv4.orig.dest_ip;
 101                         hv3 = e->ipv4.orig.src_ip;
 102                         break;
 103                 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
 104                 case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
 105                         hv1 = e->ipv6.src_ip[3] ^ e->ipv6.dest_ip[3];
 106                         hv1 ^= e->ipv6.ports;
 107
 108                         hv2 = e->ipv6.src_ip[2] ^ e->ipv6.dest_ip[2];
 109                         hv2 ^= e->ipv6.dest_ip[0];
 110
 111                         hv3 = e->ipv6.src_ip[1] ^ e->ipv6.dest_ip[1];
 112                         hv3 ^= e->ipv6.src_ip[0];
 113                         break;
 114                 case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 115                 case MTK_PPE_PKT_TYPE_IPV6_6RD:
 116                 default:
 117                         WARN_ON_ONCE(1);
 118                         return MTK_PPE_HASH_MASK;
 119         }
 120
 121         hash = (hv1 & hv2) | ((~hv1) & hv3);
 122         hash = (hash >> 24) | ((hash & 0xffffff) << 8);
 123         hash ^= hv1 ^ hv2 ^ hv3;
 124         hash ^= hash >> 16;
 125         hash <<= (ffs(eth->soc->hash_offset) - 1);
 126         hash &= MTK_PPE_ENTRIES - 1;
 127
 128         return hash;
 129 }
 130
 131 static inline struct mtk_foe_mac_info *
 132 mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
 133 {
 134         int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 135
 136         if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 137                 return &entry->bridge.l2;
 138
 139         if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 140                 return &entry->ipv6.l2;
 141
 142         return &entry->ipv4.l2;
 143 }
 144
 145 static inline u32 *
 146 mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
 147 {
 148         int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 149
 150         if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 151                 return &entry->bridge.ib2;
 152
 153         if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 154                 return &entry->ipv6.ib2;
 155
 156         return &entry->ipv4.ib2;
 157 }
 158
 159 int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry,
 160                           int type, int l4proto, u8 pse_port, u8 *src_mac,
 161                           u8 *dest_mac)
 162 {
 163         struct mtk_foe_mac_info *l2;
 164         u32 ports_pad, val;
 165
 166         memset(entry, 0, sizeof(*entry));
 167
 168         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 169                 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
 170                       FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) |
 171                       FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
 172                       MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2;
 173                 entry->ib1 = val;
 174
 175                 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) |
 176                       FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf);
 177         } else {
 178                 val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
 179                       FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
 180                       FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
 181                       MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL;
 182                 entry->ib1 = val;
 183
 184                 val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) |
 185                       FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) |
 186                       FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f);
 187         }
 188
 189         if (is_multicast_ether_addr(dest_mac))
 190                 val |= mtk_get_ib2_multicast_mask(eth);
 191
 192         ports_pad = 0xa5a5a500 | (l4proto & 0xff);
 193         if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
 194                 entry->ipv4.orig.ports = ports_pad;
 195         if (type == MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
 196                 entry->ipv6.ports = ports_pad;
 197
 198         if (type == MTK_PPE_PKT_TYPE_BRIDGE) {
 199                 ether_addr_copy(entry->bridge.src_mac, src_mac);
 200                 ether_addr_copy(entry->bridge.dest_mac, dest_mac);
 201                 entry->bridge.ib2 = val;
 202                 l2 = &entry->bridge.l2;
 203         } else if (type >= MTK_PPE_PKT_TYPE_IPV4_DSLITE) {
 204                 entry->ipv6.ib2 = val;
 205                 l2 = &entry->ipv6.l2;
 206         } else {
 207                 entry->ipv4.ib2 = val;
 208                 l2 = &entry->ipv4.l2;
 209         }
 210
 211         l2->dest_mac_hi = get_unaligned_be32(dest_mac);
 212         l2->dest_mac_lo = get_unaligned_be16(dest_mac + 4);
 213         l2->src_mac_hi = get_unaligned_be32(src_mac);
 214         l2->src_mac_lo = get_unaligned_be16(src_mac + 4);
 215
 216         if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T)
 217                 l2->etype = ETH_P_IPV6;
 218         else
 219                 l2->etype = ETH_P_IP;
 220
 221         return 0;
 222 }
 223
 224 int mtk_foe_entry_set_pse_port(struct mtk_eth *eth,
 225                                struct mtk_foe_entry *entry, u8 port)
 226 {
 227         u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
 228         u32 val = *ib2;
 229
 230         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 231                 val &= ~MTK_FOE_IB2_DEST_PORT_V2;
 232                 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port);
 233         } else {
 234                 val &= ~MTK_FOE_IB2_DEST_PORT;
 235                 val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
 236         }
 237         *ib2 = val;
 238
 239         return 0;
 240 }
 241
 242 int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth,
 243                                  struct mtk_foe_entry *entry, bool egress,
 244                                  __be32 src_addr, __be16 src_port,
 245                                  __be32 dest_addr, __be16 dest_port)
 246 {
 247         int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 248         struct mtk_ipv4_tuple *t;
 249
 250         switch (type) {
 251         case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
 252                 if (egress) {
 253                         t = &entry->ipv4.new;
 254                         break;
 255                 }
 256                 fallthrough;
 257         case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 258         case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
 259                 t = &entry->ipv4.orig;
 260                 break;
 261         case MTK_PPE_PKT_TYPE_IPV6_6RD:
 262                 entry->ipv6_6rd.tunnel_src_ip = be32_to_cpu(src_addr);
 263                 entry->ipv6_6rd.tunnel_dest_ip = be32_to_cpu(dest_addr);
 264                 return 0;
 265         default:
 266                 WARN_ON_ONCE(1);
 267                 return -EINVAL;
 268         }
 269
 270         t->src_ip = be32_to_cpu(src_addr);
 271         t->dest_ip = be32_to_cpu(dest_addr);
 272
 273         if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
 274                 return 0;
 275
 276         t->src_port = be16_to_cpu(src_port);
 277         t->dest_port = be16_to_cpu(dest_port);
 278
 279         return 0;
 280 }
 281
 282 int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth,
 283                                  struct mtk_foe_entry *entry,
 284                                  __be32 *src_addr, __be16 src_port,
 285                                  __be32 *dest_addr, __be16 dest_port)
 286 {
 287         int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 288         u32 *src, *dest;
 289         int i;
 290
 291         switch (type) {
 292         case MTK_PPE_PKT_TYPE_IPV4_DSLITE:
 293                 src = entry->dslite.tunnel_src_ip;
 294                 dest = entry->dslite.tunnel_dest_ip;
 295                 break;
 296         case MTK_PPE_PKT_TYPE_IPV6_ROUTE_5T:
 297         case MTK_PPE_PKT_TYPE_IPV6_6RD:
 298                 entry->ipv6.src_port = be16_to_cpu(src_port);
 299                 entry->ipv6.dest_port = be16_to_cpu(dest_port);
 300                 fallthrough;
 301         case MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T:
 302                 src = entry->ipv6.src_ip;
 303                 dest = entry->ipv6.dest_ip;
 304                 break;
 305         default:
 306                 WARN_ON_ONCE(1);
 307                 return -EINVAL;
 308         }
 309
 310         for (i = 0; i < 4; i++)
 311                 src[i] = be32_to_cpu(src_addr[i]);
 312         for (i = 0; i < 4; i++)
 313                 dest[i] = be32_to_cpu(dest_addr[i]);
 314
 315         return 0;
 316 }
 317
 318 int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry,
 319                           int port)
 320 {
 321         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 322
 323         l2->etype = BIT(port);
 324
 325         if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)))
 326                 entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1);
 327         else
 328                 l2->etype |= BIT(8);
 329
 330         entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth);
 331
 332         return 0;
 333 }
 334
 335 int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry,
 336                            int vid)
 337 {
 338         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 339
 340         switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) {
 341         case 0:
 342                 entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) |
 343                               mtk_prep_ib1_vlan_layer(eth, 1);
 344                 l2->vlan1 = vid;
 345                 return 0;
 346         case 1:
 347                 if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) {
 348                         l2->vlan1 = vid;
 349                         l2->etype |= BIT(8);
 350                 } else {
 351                         l2->vlan2 = vid;
 352                         entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1);
 353                 }
 354                 return 0;
 355         default:
 356                 return -ENOSPC;
 357         }
 358 }
 359
 360 int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry,
 361                             int sid)
 362 {
 363         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 364
 365         if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) ||
 366             (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth)))
 367                 l2->etype = ETH_P_PPP_SES;
 368
 369         entry->ib1 |= mtk_get_ib1_ppoe_mask(eth);
 370         l2->pppoe_id = sid;
 371
 372         return 0;
 373 }
 374
 375 int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry,
 376                            int wdma_idx, int txq, int bss, int wcid)
 377 {
 378         struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 379         u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
 380
 381         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 382                 *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2;
 383                 *ib2 |=  FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) |
 384                          MTK_FOE_IB2_WDMA_WINFO_V2;
 385                 l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) |
 386                             FIELD_PREP(MTK_FOE_WINFO_BSS, bss);
 387         } else {
 388                 *ib2 &= ~MTK_FOE_IB2_PORT_MG;
 389                 *ib2 |= MTK_FOE_IB2_WDMA_WINFO;
 390                 if (wdma_idx)
 391                         *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
 392                 l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
 393                             FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
 394                             FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
 395         }
 396
 397         return 0;
 398 }
 399
 400 static bool
 401 mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry,
 402                      struct mtk_foe_entry *data)
 403 {
 404         int type, len;
 405
 406         if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
 407                 return false;
 408
 409         type = mtk_get_ib1_pkt_type(eth, entry->data.ib1);
 410         if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 411                 len = offsetof(struct mtk_foe_entry, ipv6._rsv);
 412         else
 413                 len = offsetof(struct mtk_foe_entry, ipv4.ib2);
 414
 415         return !memcmp(&entry->data.data, &data->data, len - 4);
 416 }
 417
 418 static void
 419 __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 420 {
 421         struct hlist_head *head;
 422         struct hlist_node *tmp;
 423
 424         if (entry->type == MTK_FLOW_TYPE_L2) {
 425                 rhashtable_remove_fast(&ppe->l2_flows, &entry->l2_node,
 426                                        mtk_flow_l2_ht_params);
 427
 428                 head = &entry->l2_flows;
 429                 hlist_for_each_entry_safe(entry, tmp, head, l2_data.list)
 430                         __mtk_foe_entry_clear(ppe, entry);
 431                 return;
 432         }
 433
 434         hlist_del_init(&entry->list);
 435         if (entry->hash != 0xffff) {
 436                 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash);
 437
 438                 hwe->ib1 &= ~MTK_FOE_IB1_STATE;
 439                 hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID);
 440                 dma_wmb();
 441         }
 442         entry->hash = 0xffff;
 443
 444         if (entry->type != MTK_FLOW_TYPE_L2_SUBFLOW)
 445                 return;
 446
 447         hlist_del_init(&entry->l2_data.list);
 448         kfree(entry);
 449 }
 450
 451 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
 452 {
 453         u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
 454         u16 now = mtk_eth_timestamp(ppe->eth);
 455         u16 timestamp = ib1 & ib1_ts_mask;
 456
 457         if (timestamp > now)
 458                 return ib1_ts_mask + 1 - timestamp + now;
 459         else
 460                 return now - timestamp;
 461 }
 462
 463 static void
 464 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 465 {
 466         u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
 467         struct mtk_flow_entry *cur;
 468         struct mtk_foe_entry *hwe;
 469         struct hlist_node *tmp;
 470         int idle;
 471
 472         idle = __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
 473         hlist_for_each_entry_safe(cur, tmp, &entry->l2_flows, l2_data.list) {
 474                 int cur_idle;
 475                 u32 ib1;
 476
 477                 hwe = mtk_foe_get_entry(ppe, cur->hash);
 478                 ib1 = READ_ONCE(hwe->ib1);
 479
 480                 if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) {
 481                         cur->hash = 0xffff;
 482                         __mtk_foe_entry_clear(ppe, cur);
 483                         continue;
 484                 }
 485
 486                 cur_idle = __mtk_foe_entry_idle_time(ppe, ib1);
 487                 if (cur_idle >= idle)
 488                         continue;
 489
 490                 idle = cur_idle;
 491                 entry->data.ib1 &= ~ib1_ts_mask;
 492                 entry->data.ib1 |= hwe->ib1 & ib1_ts_mask;
 493         }
 494 }
 495
 496 static void
 497 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 498 {
 499         struct mtk_foe_entry foe = {};
 500         struct mtk_foe_entry *hwe;
 501
 502         spin_lock_bh(&ppe_lock);
 503
 504         if (entry->type == MTK_FLOW_TYPE_L2) {
 505                 mtk_flow_entry_update_l2(ppe, entry);
 506                 goto out;
 507         }
 508
 509         if (entry->hash == 0xffff)
 510                 goto out;
 511
 512         hwe = mtk_foe_get_entry(ppe, entry->hash);
 513         memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size);
 514         if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) {
 515                 entry->hash = 0xffff;
 516                 goto out;
 517         }
 518
 519         entry->data.ib1 = foe.ib1;
 520
 521 out:
 522         spin_unlock_bh(&ppe_lock);
 523 }
 524
 525 static void
 526 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
 527                        u16 hash)
 528 {
 529         struct mtk_eth *eth = ppe->eth;
 530         u16 timestamp = mtk_eth_timestamp(eth);
 531         struct mtk_foe_entry *hwe;
 532
 533         if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
 534                 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2;
 535                 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2,
 536                                          timestamp);
 537         } else {
 538                 entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
 539                 entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP,
 540                                          timestamp);
 541         }
 542
 543         hwe = mtk_foe_get_entry(ppe, hash);
 544         memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1));
 545         wmb();
 546         hwe->ib1 = entry->ib1;
 547
 548         dma_wmb();
 549
 550         mtk_ppe_cache_clear(ppe);
 551 }
 552
 553 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 554 {
 555         spin_lock_bh(&ppe_lock);
 556         __mtk_foe_entry_clear(ppe, entry);
 557         spin_unlock_bh(&ppe_lock);
 558 }
 559
 560 static int
 561 mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 562 {
 563         entry->type = MTK_FLOW_TYPE_L2;
 564
 565         return rhashtable_insert_fast(&ppe->l2_flows, &entry->l2_node,
 566                                       mtk_flow_l2_ht_params);
 567 }
 568
 569 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 570 {
 571         const struct mtk_soc_data *soc = ppe->eth->soc;
 572         int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1);
 573         u32 hash;
 574
 575         if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 576                 return mtk_foe_entry_commit_l2(ppe, entry);
 577
 578         hash = mtk_ppe_hash_entry(ppe->eth, &entry->data);
 579         entry->hash = 0xffff;
 580         spin_lock_bh(&ppe_lock);
 581         hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]);
 582         spin_unlock_bh(&ppe_lock);
 583
 584         return 0;
 585 }
 586
 587 static void
 588 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
 589                              u16 hash)
 590 {
 591         const struct mtk_soc_data *soc = ppe->eth->soc;
 592         struct mtk_flow_entry *flow_info;
 593         struct mtk_foe_entry foe = {}, *hwe;
 594         struct mtk_foe_mac_info *l2;
 595         u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP;
 596         int type;
 597
 598         flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end),
 599                             GFP_ATOMIC);
 600         if (!flow_info)
 601                 return;
 602
 603         flow_info->l2_data.base_flow = entry;
 604         flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW;
 605         flow_info->hash = hash;
 606         hlist_add_head(&flow_info->list,
 607                        &ppe->foe_flow[hash / soc->hash_offset]);
 608         hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows);
 609
 610         hwe = mtk_foe_get_entry(ppe, hash);
 611         memcpy(&foe, hwe, soc->foe_entry_size);
 612         foe.ib1 &= ib1_mask;
 613         foe.ib1 |= entry->data.ib1 & ~ib1_mask;
 614
 615         l2 = mtk_foe_entry_l2(ppe->eth, &foe);
 616         memcpy(l2, &entry->data.bridge.l2, sizeof(*l2));
 617
 618         type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1);
 619         if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT)
 620                 memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new));
 621         else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP)
 622                 l2->etype = ETH_P_IPV6;
 623
 624         *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2;
 625
 626         __mtk_foe_entry_commit(ppe, &foe, hash);
 627 }
 628
 629 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
 630 {
 631         const struct mtk_soc_data *soc = ppe->eth->soc;
 632         struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset];
 633         struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash);
 634         struct mtk_flow_entry *entry;
 635         struct mtk_foe_bridge key = {};
 636         struct hlist_node *n;
 637         struct ethhdr *eh;
 638         bool found = false;
 639         u8 *tag;
 640
 641         spin_lock_bh(&ppe_lock);
 642
 643         if (FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) == MTK_FOE_STATE_BIND)
 644                 goto out;
 645
 646         hlist_for_each_entry_safe(entry, n, head, list) {
 647                 if (entry->type == MTK_FLOW_TYPE_L2_SUBFLOW) {
 648                         if (unlikely(FIELD_GET(MTK_FOE_IB1_STATE, hwe->ib1) ==
 649                                      MTK_FOE_STATE_BIND))
 650                                 continue;
 651
 652                         entry->hash = 0xffff;
 653                         __mtk_foe_entry_clear(ppe, entry);
 654                         continue;
 655                 }
 656
 657                 if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) {
 658                         if (entry->hash != 0xffff)
 659                                 entry->hash = 0xffff;
 660                         continue;
 661                 }
 662
 663                 entry->hash = hash;
 664                 __mtk_foe_entry_commit(ppe, &entry->data, hash);
 665                 found = true;
 666         }
 667
 668         if (found)
 669                 goto out;
 670
 671         eh = eth_hdr(skb);
 672         ether_addr_copy(key.dest_mac, eh->h_dest);
 673         ether_addr_copy(key.src_mac, eh->h_source);
 674         tag = skb->data - 2;
 675         key.vlan = 0;
 676         switch (skb->protocol) {
 677 #if IS_ENABLED(CONFIG_NET_DSA)
 678         case htons(ETH_P_XDSA):
 679                 if (!netdev_uses_dsa(skb->dev) ||
 680                     skb->dev->dsa_ptr->tag_ops->proto != DSA_TAG_PROTO_MTK)
 681                         goto out;
 682
 683                 tag += 4;
 684                 if (get_unaligned_be16(tag) != ETH_P_8021Q)
 685                         break;
 686
 687                 fallthrough;
 688 #endif
 689         case htons(ETH_P_8021Q):
 690                 key.vlan = get_unaligned_be16(tag + 2) & VLAN_VID_MASK;
 691                 break;
 692         default:
 693                 break;
 694         }
 695
 696         entry = rhashtable_lookup_fast(&ppe->l2_flows, &key, mtk_flow_l2_ht_params);
 697         if (!entry)
 698                 goto out;
 699
 700         mtk_foe_entry_commit_subflow(ppe, entry, hash);
 701
 702 out:
 703         spin_unlock_bh(&ppe_lock);
 704 }
 705
 706 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 707 {
 708         mtk_flow_entry_update(ppe, entry);
 709
 710         return __mtk_foe_entry_idle_time(ppe, entry->data.ib1);
 711 }
 712
 713 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
 714                              int version, int index)
 715 {
 716         const struct mtk_soc_data *soc = eth->soc;
 717         struct device *dev = eth->dev;
 718         struct mtk_ppe *ppe;
 719         u32 foe_flow_size;
 720         void *foe;
 721
 722         ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
 723         if (!ppe)
 724                 return NULL;
 725
 726         rhashtable_init(&ppe->l2_flows, &mtk_flow_l2_ht_params);
 727
 728         /* need to allocate a separate device, since it PPE DMA access is
 729          * not coherent.
 730          */
 731         ppe->base = base;
 732         ppe->eth = eth;
 733         ppe->dev = dev;
 734         ppe->version = version;
 735
 736         foe = dmam_alloc_coherent(ppe->dev,
 737                                   MTK_PPE_ENTRIES * soc->foe_entry_size,
 738                                   &ppe->foe_phys, GFP_KERNEL);
 739         if (!foe)
 740                 return NULL;
 741
 742         ppe->foe_table = foe;
 743
 744         foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) *
 745                         sizeof(*ppe->foe_flow);
 746         ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL);
 747         if (!ppe->foe_flow)
 748                 return NULL;
 749
 750         mtk_ppe_debugfs_init(ppe, index);
 751
 752         return ppe;
 753 }
 754
 755 static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
 756 {
 757         static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
 758         int i, k;
 759
 760         memset(ppe->foe_table, 0,
 761                MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size);
 762
 763         if (!IS_ENABLED(CONFIG_SOC_MT7621))
 764                 return;
 765
 766         /* skip all entries that cross the 1024 byte boundary */
 767         for (i = 0; i < MTK_PPE_ENTRIES; i += 128) {
 768                 for (k = 0; k < ARRAY_SIZE(skip); k++) {
 769                         struct mtk_foe_entry *hwe;
 770
 771                         hwe = mtk_foe_get_entry(ppe, i + skip[k]);
 772                         hwe->ib1 |= MTK_FOE_IB1_STATIC;
 773                 }
 774         }
 775 }
 776
 777 void mtk_ppe_start(struct mtk_ppe *ppe)
 778 {
 779         u32 val;
 780
 781         if (!ppe)
 782                 return;
 783
 784         mtk_ppe_init_foe_table(ppe);
 785         ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
 786
 787         val = MTK_PPE_TB_CFG_ENTRY_80B |
 788               MTK_PPE_TB_CFG_AGE_NON_L4 |
 789               MTK_PPE_TB_CFG_AGE_UNBIND |
 790               MTK_PPE_TB_CFG_AGE_TCP |
 791               MTK_PPE_TB_CFG_AGE_UDP |
 792               MTK_PPE_TB_CFG_AGE_TCP_FIN |
 793               FIELD_PREP(MTK_PPE_TB_CFG_SEARCH_MISS,
 794                          MTK_PPE_SEARCH_MISS_ACTION_FORWARD_BUILD) |
 795               FIELD_PREP(MTK_PPE_TB_CFG_KEEPALIVE,
 796                          MTK_PPE_KEEPALIVE_DISABLE) |
 797               FIELD_PREP(MTK_PPE_TB_CFG_HASH_MODE, 1) |
 798               FIELD_PREP(MTK_PPE_TB_CFG_SCAN_MODE,
 799                          MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
 800               FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
 801                          MTK_PPE_ENTRIES_SHIFT);
 802         if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
 803                 val |= MTK_PPE_TB_CFG_INFO_SEL;
 804         ppe_w32(ppe, MTK_PPE_TB_CFG, val);
 805
 806         ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
 807                 MTK_PPE_IP_PROTO_CHK_IPV4 | MTK_PPE_IP_PROTO_CHK_IPV6);
 808
 809         mtk_ppe_cache_enable(ppe, true);
 810
 811         val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
 812               MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
 813               MTK_PPE_FLOW_CFG_IP6_6RD |
 814               MTK_PPE_FLOW_CFG_IP4_NAT |
 815               MTK_PPE_FLOW_CFG_IP4_NAPT |
 816               MTK_PPE_FLOW_CFG_IP4_DSLITE |
 817               MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
 818         if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
 819                 val |= MTK_PPE_MD_TOAP_BYP_CRSN0 |
 820                        MTK_PPE_MD_TOAP_BYP_CRSN1 |
 821                        MTK_PPE_MD_TOAP_BYP_CRSN2 |
 822                        MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY;
 823         else
 824                 val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
 825                        MTK_PPE_FLOW_CFG_IP4_UDP_FRAG;
 826         ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
 827
 828         val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
 829               FIELD_PREP(MTK_PPE_UNBIND_AGE_DELTA, 3);
 830         ppe_w32(ppe, MTK_PPE_UNBIND_AGE, val);
 831
 832         val = FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_UDP, 12) |
 833               FIELD_PREP(MTK_PPE_BIND_AGE0_DELTA_NON_L4, 1);
 834         ppe_w32(ppe, MTK_PPE_BIND_AGE0, val);
 835
 836         val = FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP_FIN, 1) |
 837               FIELD_PREP(MTK_PPE_BIND_AGE1_DELTA_TCP, 7);
 838         ppe_w32(ppe, MTK_PPE_BIND_AGE1, val);
 839
 840         val = MTK_PPE_BIND_LIMIT0_QUARTER | MTK_PPE_BIND_LIMIT0_HALF;
 841         ppe_w32(ppe, MTK_PPE_BIND_LIMIT0, val);
 842
 843         val = MTK_PPE_BIND_LIMIT1_FULL |
 844               FIELD_PREP(MTK_PPE_BIND_LIMIT1_NON_L4, 1);
 845         ppe_w32(ppe, MTK_PPE_BIND_LIMIT1, val);
 846
 847         val = FIELD_PREP(MTK_PPE_BIND_RATE_BIND, 30) |
 848               FIELD_PREP(MTK_PPE_BIND_RATE_PREBIND, 1);
 849         ppe_w32(ppe, MTK_PPE_BIND_RATE, val);
 850
 851         /* enable PPE */
 852         val = MTK_PPE_GLO_CFG_EN |
 853               MTK_PPE_GLO_CFG_IP4_L4_CS_DROP |
 854               MTK_PPE_GLO_CFG_IP4_CS_DROP |
 855               MTK_PPE_GLO_CFG_FLOW_DROP_UPDATE;
 856         ppe_w32(ppe, MTK_PPE_GLO_CFG, val);
 857
 858         ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
 859
 860         if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) {
 861                 ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);
 862                 ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);
 863         }
 864 }
 865
 866 int mtk_ppe_stop(struct mtk_ppe *ppe)
 867 {
 868         u32 val;
 869         int i;
 870
 871         if (!ppe)
 872                 return 0;
 873
 874         for (i = 0; i < MTK_PPE_ENTRIES; i++) {
 875                 struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i);
 876
 877                 hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
 878                                       MTK_FOE_STATE_INVALID);
 879         }
 880
 881         mtk_ppe_cache_enable(ppe, false);
 882
 883         /* disable offload engine */
 884         ppe_clear(ppe, MTK_PPE_GLO_CFG, MTK_PPE_GLO_CFG_EN);
 885         ppe_w32(ppe, MTK_PPE_FLOW_CFG, 0);
 886
 887         /* disable aging */
 888         val = MTK_PPE_TB_CFG_AGE_NON_L4 |
 889               MTK_PPE_TB_CFG_AGE_UNBIND |
 890               MTK_PPE_TB_CFG_AGE_TCP |
 891               MTK_PPE_TB_CFG_AGE_UDP |
 892               MTK_PPE_TB_CFG_AGE_TCP_FIN;
 893         ppe_clear(ppe, MTK_PPE_TB_CFG, val);
 894
 895         return mtk_ppe_wait_busy(ppe);
 896 }