summaryrefslogtreecommitdiffstats
path: root/drivers/net/ethernet/sfc
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/sfc')
-rw-r--r--drivers/net/ethernet/sfc/Makefile2
-rw-r--r--drivers/net/ethernet/sfc/ef100_ethtool.c2
-rw-r--r--drivers/net/ethernet/sfc/ef100_rx.c23
-rw-r--r--drivers/net/ethernet/sfc/ef100_tx.c3
-rw-r--r--drivers/net/ethernet/sfc/efx_channels.c9
-rw-r--r--drivers/net/ethernet/sfc/ethtool_common.c37
-rw-r--r--drivers/net/ethernet/sfc/ethtool_common.h2
-rw-r--r--drivers/net/ethernet/sfc/mae.c306
-rw-r--r--drivers/net/ethernet/sfc/mae.h7
-rw-r--r--drivers/net/ethernet/sfc/mae_counter_format.h73
-rw-r--r--drivers/net/ethernet/sfc/mcdi.h17
-rw-r--r--drivers/net/ethernet/sfc/net_driver.h19
-rw-r--r--drivers/net/ethernet/sfc/ptp.c7
-rw-r--r--drivers/net/ethernet/sfc/rx_common.c3
-rw-r--r--drivers/net/ethernet/sfc/siena/ptp.c7
-rw-r--r--drivers/net/ethernet/sfc/tc.c302
-rw-r--r--drivers/net/ethernet/sfc/tc.h48
-rw-r--r--drivers/net/ethernet/sfc/tc_counters.c503
-rw-r--r--drivers/net/ethernet/sfc/tc_counters.h59
-rw-r--r--drivers/net/ethernet/sfc/tx.c4
20 files changed, 1317 insertions, 116 deletions
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile
index b5e45fc6337e..712a48d00069 100644
--- a/drivers/net/ethernet/sfc/Makefile
+++ b/drivers/net/ethernet/sfc/Makefile
@@ -9,7 +9,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \
ef100_ethtool.o ef100_rx.o ef100_tx.o
sfc-$(CONFIG_SFC_MTD) += mtd.o
sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \
- mae.o tc.o tc_bindings.o
+ mae.o tc.o tc_bindings.o tc_counters.o
obj-$(CONFIG_SFC) += sfc.o
diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c
index 135ece2f1375..702abbe59b76 100644
--- a/drivers/net/ethernet/sfc/ef100_ethtool.c
+++ b/drivers/net/ethernet/sfc/ef100_ethtool.c
@@ -43,8 +43,6 @@ const struct ethtool_ops ef100_ethtool_ops = {
.get_pauseparam = efx_ethtool_get_pauseparam,
.set_pauseparam = efx_ethtool_set_pauseparam,
.get_sset_count = efx_ethtool_get_sset_count,
- .get_priv_flags = efx_ethtool_get_priv_flags,
- .set_priv_flags = efx_ethtool_set_priv_flags,
.self_test = efx_ethtool_self_test,
.get_strings = efx_ethtool_get_strings,
.get_link_ksettings = efx_ethtool_get_link_ksettings,
diff --git a/drivers/net/ethernet/sfc/ef100_rx.c b/drivers/net/ethernet/sfc/ef100_rx.c
index 65bbe37753e6..83d9db71d7d7 100644
--- a/drivers/net/ethernet/sfc/ef100_rx.c
+++ b/drivers/net/ethernet/sfc/ef100_rx.c
@@ -21,7 +21,7 @@
/* Get the value of a field in the RX prefix */
#define PREFIX_OFFSET_W(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN / 32)
#define PREFIX_OFFSET_B(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN % 32)
-#define PREFIX_WIDTH_MASK(_f) ((1UL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)
+#define PREFIX_WIDTH_MASK(_f) ((1ULL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)
#define PREFIX_WORD(_p, _f) le32_to_cpu((__force __le32)(_p)[PREFIX_OFFSET_W(_f)])
#define PREFIX_FIELD(_p, _f) ((PREFIX_WORD(_p, _f) >> PREFIX_OFFSET_B(_f)) & \
PREFIX_WIDTH_MASK(_f))
@@ -67,6 +67,13 @@ void __ef100_rx_packet(struct efx_channel *channel)
prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN);
+ if (channel->type->receive_raw) {
+ u32 mark = PREFIX_FIELD(prefix, USER_MARK);
+
+ if (channel->type->receive_raw(rx_queue, mark))
+ return; /* packet was consumed */
+ }
+
if (ef100_has_fcs_error(channel, prefix) &&
unlikely(!(efx->net_dev->features & NETIF_F_RXALL)))
goto out;
@@ -183,24 +190,32 @@ void efx_ef100_ev_rx(struct efx_channel *channel, const efx_qword_t *p_event)
void ef100_rx_write(struct efx_rx_queue *rx_queue)
{
+ unsigned int notified_count = rx_queue->notified_count;
struct efx_rx_buffer *rx_buf;
unsigned int idx;
efx_qword_t *rxd;
efx_dword_t rxdb;
- while (rx_queue->notified_count != rx_queue->added_count) {
- idx = rx_queue->notified_count & rx_queue->ptr_mask;
+ while (notified_count != rx_queue->added_count) {
+ idx = notified_count & rx_queue->ptr_mask;
rx_buf = efx_rx_buffer(rx_queue, idx);
rxd = efx_rx_desc(rx_queue, idx);
EFX_POPULATE_QWORD_1(*rxd, ESF_GZ_RX_BUF_ADDR, rx_buf->dma_addr);
- ++rx_queue->notified_count;
+ ++notified_count;
}
+ if (notified_count == rx_queue->notified_count)
+ return;
wmb();
EFX_POPULATE_DWORD_1(rxdb, ERF_GZ_RX_RING_PIDX,
rx_queue->added_count & rx_queue->ptr_mask);
efx_writed_page(rx_queue->efx, &rxdb,
ER_GZ_RX_RING_DOORBELL, efx_rx_queue_index(rx_queue));
+ if (rx_queue->grant_credits)
+ wmb();
+ rx_queue->notified_count = notified_count;
+ if (rx_queue->grant_credits)
+ schedule_work(&rx_queue->grant_work);
}
diff --git a/drivers/net/ethernet/sfc/ef100_tx.c b/drivers/net/ethernet/sfc/ef100_tx.c
index 102ddc7e206a..29ffaf35559d 100644
--- a/drivers/net/ethernet/sfc/ef100_tx.c
+++ b/drivers/net/ethernet/sfc/ef100_tx.c
@@ -367,7 +367,8 @@ void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event)
* Returns 0 on success, error code otherwise. In case of an error this
* function will free the SKB.
*/
-int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
+netdev_tx_t ef100_enqueue_skb(struct efx_tx_queue *tx_queue,
+ struct sk_buff *skb)
{
return __ef100_enqueue_skb(tx_queue, skb, NULL);
}
diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c
index aaa381743bca..fcea3ea809d7 100644
--- a/drivers/net/ethernet/sfc/efx_channels.c
+++ b/drivers/net/ethernet/sfc/efx_channels.c
@@ -1119,6 +1119,8 @@ void efx_start_channels(struct efx_nic *efx)
struct efx_channel *channel;
efx_for_each_channel_rev(channel, efx) {
+ if (channel->type->start)
+ channel->type->start(channel);
efx_for_each_channel_tx_queue(tx_queue, channel) {
efx_init_tx_queue(tx_queue);
atomic_inc(&efx->active_queues);
@@ -1143,8 +1145,13 @@ void efx_stop_channels(struct efx_nic *efx)
struct efx_channel *channel;
int rc = 0;
- /* Stop RX refill */
+ /* Stop special channels and RX refill.
+ * The channel's stop has to be called first, since it might wait
+ * for a sentinel RX to indicate the channel has fully drained.
+ */
efx_for_each_channel(channel, efx) {
+ if (channel->type->stop)
+ channel->type->stop(channel);
efx_for_each_channel_rx_queue(rx_queue, channel)
rx_queue->refill_enabled = false;
}
diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c
index 6649a2327d03..a8cbceeb301b 100644
--- a/drivers/net/ethernet/sfc/ethtool_common.c
+++ b/drivers/net/ethernet/sfc/ethtool_common.c
@@ -101,14 +101,6 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = {
#define EFX_ETHTOOL_SW_STAT_COUNT ARRAY_SIZE(efx_sw_stat_desc)
-static const char efx_ethtool_priv_flags_strings[][ETH_GSTRING_LEN] = {
- "log-tc-errors",
-};
-
-#define EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS BIT(0)
-
-#define EFX_ETHTOOL_PRIV_FLAGS_COUNT ARRAY_SIZE(efx_ethtool_priv_flags_strings)
-
void efx_ethtool_get_drvinfo(struct net_device *net_dev,
struct ethtool_drvinfo *info)
{
@@ -460,8 +452,6 @@ int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set)
efx_ptp_describe_stats(efx, NULL);
case ETH_SS_TEST:
return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL);
- case ETH_SS_PRIV_FLAGS:
- return EFX_ETHTOOL_PRIV_FLAGS_COUNT;
default:
return -EINVAL;
}
@@ -488,39 +478,12 @@ void efx_ethtool_get_strings(struct net_device *net_dev,
case ETH_SS_TEST:
efx_ethtool_fill_self_tests(efx, NULL, strings, NULL);
break;
- case ETH_SS_PRIV_FLAGS:
- for (i = 0; i < EFX_ETHTOOL_PRIV_FLAGS_COUNT; i++)
- strscpy(strings + i * ETH_GSTRING_LEN,
- efx_ethtool_priv_flags_strings[i],
- ETH_GSTRING_LEN);
- break;
default:
/* No other string sets */
break;
}
}
-u32 efx_ethtool_get_priv_flags(struct net_device *net_dev)
-{
- struct efx_nic *efx = efx_netdev_priv(net_dev);
- u32 ret_flags = 0;
-
- if (efx->log_tc_errs)
- ret_flags |= EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS;
-
- return ret_flags;
-}
-
-int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags)
-{
- struct efx_nic *efx = efx_netdev_priv(net_dev);
-
- efx->log_tc_errs =
- !!(flags & EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS);
-
- return 0;
-}
-
void efx_ethtool_get_stats(struct net_device *net_dev,
struct ethtool_stats *stats,
u64 *data)
diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h
index 0afc74021a5e..659491932101 100644
--- a/drivers/net/ethernet/sfc/ethtool_common.h
+++ b/drivers/net/ethernet/sfc/ethtool_common.h
@@ -27,8 +27,6 @@ int efx_ethtool_fill_self_tests(struct efx_nic *efx,
int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set);
void efx_ethtool_get_strings(struct net_device *net_dev, u32 string_set,
u8 *strings);
-u32 efx_ethtool_get_priv_flags(struct net_device *net_dev);
-int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags);
void efx_ethtool_get_stats(struct net_device *net_dev,
struct ethtool_stats *stats __attribute__ ((unused)),
u64 *data);
diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c
index 874c765b2465..583baf69981c 100644
--- a/drivers/net/ethernet/sfc/mae.c
+++ b/drivers/net/ethernet/sfc/mae.c
@@ -112,6 +112,117 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id)
return 0;
}
+int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_LEN);
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN);
+ u32 out_flags;
+ size_t outlen;
+ int rc;
+
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_QID,
+ efx_rx_queue_index(rx_queue));
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE,
+ efx->net_dev->mtu);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK,
+ BIT(MAE_COUNTER_TYPE_AR) | BIT(MAE_COUNTER_TYPE_CT) |
+ BIT(MAE_COUNTER_TYPE_OR));
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_START,
+ inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ out_flags = MCDI_DWORD(outbuf, MAE_COUNTERS_STREAM_START_OUT_FLAGS);
+ if (out_flags & BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST)) {
+ netif_dbg(efx, drv, efx->net_dev,
+ "MAE counter stream uses credits\n");
+ rx_queue->grant_credits = true;
+ out_flags &= ~BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST);
+ }
+ if (out_flags) {
+ netif_err(efx, drv, efx->net_dev,
+ "MAE counter stream start: unrecognised flags %x\n",
+ out_flags);
+ goto out_stop;
+ }
+ return 0;
+out_stop:
+ efx_mae_stop_counters(efx, rx_queue);
+ return -EOPNOTSUPP;
+}
+
+static bool efx_mae_counters_flushed(u32 *flush_gen, u32 *seen_gen)
+{
+ int i;
+
+ for (i = 0; i < EFX_TC_COUNTER_TYPE_MAX; i++)
+ if ((s32)(flush_gen[i] - seen_gen[i]) > 0)
+ return false;
+ return true;
+}
+
+int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX);
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN);
+ size_t outlen;
+ int rc, i;
+
+ MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_STOP_IN_QID,
+ efx_rx_queue_index(rx_queue));
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_STOP,
+ inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
+
+ if (rc)
+ return rc;
+
+ netif_dbg(efx, drv, efx->net_dev, "Draining counters:\n");
+ /* Only process received generation counts */
+ for (i = 0; (i < (outlen / 4)) && (i < EFX_TC_COUNTER_TYPE_MAX); i++) {
+ efx->tc->flush_gen[i] = MCDI_ARRAY_DWORD(outbuf,
+ MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT,
+ i);
+ netif_dbg(efx, drv, efx->net_dev,
+ "\ttype %u, awaiting gen %u\n", i,
+ efx->tc->flush_gen[i]);
+ }
+
+ efx->tc->flush_counters = true;
+
+ /* Drain can take up to 2 seconds owing to FWRIVERHD-2884; whatever
+ * timeout we use, that delay is added to unload on nonresponsive
+ * hardware, so 2500ms seems like a reasonable compromise.
+ */
+ if (!wait_event_timeout(efx->tc->flush_wq,
+ efx_mae_counters_flushed(efx->tc->flush_gen,
+ efx->tc->seen_gen),
+ msecs_to_jiffies(2500)))
+ netif_warn(efx, drv, efx->net_dev,
+ "Failed to drain counters RXQ, FW may be unhappy\n");
+
+ efx->tc->flush_counters = false;
+
+ return rc;
+}
+
+void efx_mae_counters_grant_credits(struct work_struct *work)
+{
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN);
+ struct efx_rx_queue *rx_queue = container_of(work, struct efx_rx_queue,
+ grant_work);
+ struct efx_nic *efx = rx_queue->efx;
+ unsigned int credits;
+
+ BUILD_BUG_ON(MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN);
+ credits = READ_ONCE(rx_queue->notified_count) - rx_queue->granted_count;
+ MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS,
+ credits);
+ if (!efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS,
+ inbuf, sizeof(inbuf), NULL, 0, NULL))
+ rx_queue->granted_count += credits;
+}
+
static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps)
{
MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN);
@@ -250,6 +361,32 @@ static int efx_mae_match_check_cap_typ(u8 support, enum mask_type typ)
}
}
+/* Validate field mask against hardware capabilities. Captures caller's 'rc' */
+#define CHECK(_mcdi, _field) ({ \
+ enum mask_type typ = classify_mask((const u8 *)&mask->_field, \
+ sizeof(mask->_field)); \
+ \
+ rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_ ## _mcdi],\
+ typ); \
+ if (rc) \
+ NL_SET_ERR_MSG_FMT_MOD(extack, \
+ "No support for %s mask in field %s", \
+ mask_type_name(typ), #_field); \
+ rc; \
+})
+/* Booleans need special handling */
+#define CHECK_BIT(_mcdi, _field) ({ \
+ enum mask_type typ = mask->_field ? MASK_ONES : MASK_ZEROES; \
+ \
+ rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_ ## _mcdi],\
+ typ); \
+ if (rc) \
+ NL_SET_ERR_MSG_FMT_MOD(extack, \
+ "No support for %s mask in field %s", \
+ mask_type_name(typ), #_field); \
+ rc; \
+})
+
int efx_mae_match_check_caps(struct efx_nic *efx,
const struct efx_tc_match_fields *mask,
struct netlink_ext_ack *extack)
@@ -265,11 +402,86 @@ int efx_mae_match_check_caps(struct efx_nic *efx,
rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_INGRESS_PORT],
ingress_port_mask_type);
if (rc) {
- efx_tc_err(efx, "No support for %s mask in field ingress_port\n",
- mask_type_name(ingress_port_mask_type));
- NL_SET_ERR_MSG_MOD(extack, "Unsupported mask type for ingress_port");
+ NL_SET_ERR_MSG_FMT_MOD(extack, "No support for %s mask in field ingress_port",
+ mask_type_name(ingress_port_mask_type));
return rc;
}
+ if (CHECK(ETHER_TYPE, eth_proto) ||
+ CHECK(VLAN0_TCI, vlan_tci[0]) ||
+ CHECK(VLAN0_PROTO, vlan_proto[0]) ||
+ CHECK(VLAN1_TCI, vlan_tci[1]) ||
+ CHECK(VLAN1_PROTO, vlan_proto[1]) ||
+ CHECK(ETH_SADDR, eth_saddr) ||
+ CHECK(ETH_DADDR, eth_daddr) ||
+ CHECK(IP_PROTO, ip_proto) ||
+ CHECK(IP_TOS, ip_tos) ||
+ CHECK(IP_TTL, ip_ttl) ||
+ CHECK(SRC_IP4, src_ip) ||
+ CHECK(DST_IP4, dst_ip) ||
+#ifdef CONFIG_IPV6
+ CHECK(SRC_IP6, src_ip6) ||
+ CHECK(DST_IP6, dst_ip6) ||
+#endif
+ CHECK(L4_SPORT, l4_sport) ||
+ CHECK(L4_DPORT, l4_dport) ||
+ CHECK(TCP_FLAGS, tcp_flags) ||
+ CHECK_BIT(IS_IP_FRAG, ip_frag) ||
+ CHECK_BIT(IP_FIRST_FRAG, ip_firstfrag) ||
+ CHECK(RECIRC_ID, recirc_id))
+ return rc;
+ return 0;
+}
+#undef CHECK_BIT
+#undef CHECK
+
+int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(1));
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_ALLOC_V2_IN_LEN);
+ size_t outlen;
+ int rc;
+
+ if (!cnt)
+ return -EINVAL;
+
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT, 1);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE, cnt->type);
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_ALLOC, inbuf, sizeof(inbuf),
+ outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ /* pcol says this can't happen, since count is 1 */
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ cnt->fw_id = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_COUNTER_ID);
+ cnt->gen = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT);
+ return 0;
+}
+
+int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)
+{
+ MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_FREE_OUT_LEN(1));
+ MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_FREE_V2_IN_LEN);
+ size_t outlen;
+ int rc;
+
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT, 1);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID, cnt->fw_id);
+ MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE, cnt->type);
+ rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_FREE, inbuf, sizeof(inbuf),
+ outbuf, sizeof(outbuf), &outlen);
+ if (rc)
+ return rc;
+ /* pcol says this can't happen, since count is 1 */
+ if (outlen < sizeof(outbuf))
+ return -EIO;
+ /* FW freed a different ID than we asked for, should also never happen.
+ * Warn because it means we've now got a different idea to the FW of
+ * what counters exist, which could cause mayhem later.
+ */
+ if (WARN_ON(MCDI_DWORD(outbuf, MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID) !=
+ cnt->fw_id))
+ return -EIO;
return 0;
}
@@ -289,8 +501,12 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act)
MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID,
MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);
- MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
- MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
+ if (act->count && !WARN_ON(!act->count->cnt))
+ MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
+ act->count->cnt->fw_id);
+ else
+ MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
+ MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID,
MC_CMD_MAE_COUNTER_LIST_ALLOC_OUT_COUNTER_LIST_ID_NULL);
MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID,
@@ -440,10 +656,90 @@ static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit),
}
MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK,
match->mask.ingress_port);
+ EFX_POPULATE_DWORD_2(*_MCDI_STRUCT_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS),
+ MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG,
+ match->value.ip_frag,
+ MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG,
+ match->value.ip_firstfrag);
+ EFX_POPULATE_DWORD_2(*_MCDI_STRUCT_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK),
+ MAE_FIELD_MASK_VALUE_PAIRS_V2_IS_IP_FRAG,
+ match->mask.ip_frag,
+ MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_FIRST_FRAG,
+ match->mask.ip_firstfrag);
MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID,
match->value.recirc_id);
MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK,
match->mask.recirc_id);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE,
+ match->value.eth_proto);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETHER_TYPE_BE_MASK,
+ match->mask.eth_proto);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE,
+ match->value.vlan_tci[0]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_TCI_BE_MASK,
+ match->mask.vlan_tci[0]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE,
+ match->value.vlan_proto[0]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN0_PROTO_BE_MASK,
+ match->mask.vlan_proto[0]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE,
+ match->value.vlan_tci[1]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_TCI_BE_MASK,
+ match->mask.vlan_tci[1]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE,
+ match->value.vlan_proto[1]);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_VLAN1_PROTO_BE_MASK,
+ match->mask.vlan_proto[1]);
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE),
+ match->value.eth_saddr, ETH_ALEN);
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_SADDR_BE_MASK),
+ match->mask.eth_saddr, ETH_ALEN);
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE),
+ match->value.eth_daddr, ETH_ALEN);
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_ETH_DADDR_BE_MASK),
+ match->mask.eth_daddr, ETH_ALEN);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO,
+ match->value.ip_proto);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_PROTO_MASK,
+ match->mask.ip_proto);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS,
+ match->value.ip_tos);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TOS_MASK,
+ match->mask.ip_tos);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL,
+ match->value.ip_ttl);
+ MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_IP_TTL_MASK,
+ match->mask.ip_ttl);
+ MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE,
+ match->value.src_ip);
+ MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP4_BE_MASK,
+ match->mask.src_ip);
+ MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE,
+ match->value.dst_ip);
+ MCDI_STRUCT_SET_DWORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP4_BE_MASK,
+ match->mask.dst_ip);
+#ifdef CONFIG_IPV6
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE),
+ &match->value.src_ip6, sizeof(struct in6_addr));
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_SRC_IP6_BE_MASK),
+ &match->mask.src_ip6, sizeof(struct in6_addr));
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE),
+ &match->value.dst_ip6, sizeof(struct in6_addr));
+ memcpy(MCDI_STRUCT_PTR(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_DST_IP6_BE_MASK),
+ &match->mask.dst_ip6, sizeof(struct in6_addr));
+#endif
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE,
+ match->value.l4_sport);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_SPORT_BE_MASK,
+ match->mask.l4_sport);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE,
+ match->value.l4_dport);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_L4_DPORT_BE_MASK,
+ match->mask.l4_dport);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE,
+ match->value.tcp_flags);
+ MCDI_STRUCT_SET_WORD_BE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_TCP_FLAGS_BE_MASK,
+ match->mask.tcp_flags);
return 0;
}
diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h
index 3e0cd238d523..72343e90e222 100644
--- a/drivers/net/ethernet/sfc/mae.h
+++ b/drivers/net/ethernet/sfc/mae.h
@@ -27,6 +27,10 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out);
int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id);
+int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);
+int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);
+void efx_mae_counters_grant_credits(struct work_struct *work);
+
#define MAE_NUM_FIELDS (MAE_FIELD_ENC_VNET_ID + 1)
struct mae_caps {
@@ -41,6 +45,9 @@ int efx_mae_match_check_caps(struct efx_nic *efx,
const struct efx_tc_match_fields *mask,
struct netlink_ext_ack *extack);
+int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
+int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);
+
int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act);
int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id);
diff --git a/drivers/net/ethernet/sfc/mae_counter_format.h b/drivers/net/ethernet/sfc/mae_counter_format.h
new file mode 100644
index 000000000000..7e252e393fbe
--- /dev/null
+++ b/drivers/net/ethernet/sfc/mae_counter_format.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2020 Xilinx, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+/* Format of counter packets (version 2) from the ef100 Match-Action Engine */
+
+#ifndef EFX_MAE_COUNTER_FORMAT_H
+#define EFX_MAE_COUNTER_FORMAT_H
+
+
+/*------------------------------------------------------------*/
+/*
+ * ER_RX_SL_PACKETISER_HEADER_WORD(160bit):
+ *
+ */
+#define ER_RX_SL_PACKETISER_HEADER_WORD_SIZE 20
+#define ER_RX_SL_PACKETISER_HEADER_WORD_WIDTH 160
+
+#define ERF_SC_PACKETISER_HEADER_VERSION_LBN 0
+#define ERF_SC_PACKETISER_HEADER_VERSION_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_VERSION_VALUE 2
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_LBN 8
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR 0
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT 1
+#define ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR 2
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_LBN 16
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT 0x4
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_LBN 24
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_WIDTH 8
+#define ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET_DEFAULT 0x14
+#define ERF_SC_PACKETISER_HEADER_INDEX_LBN 32
+#define ERF_SC_PACKETISER_HEADER_INDEX_WIDTH 16
+#define ERF_SC_PACKETISER_HEADER_COUNT_LBN 48
+#define ERF_SC_PACKETISER_HEADER_COUNT_WIDTH 16
+#define ERF_SC_PACKETISER_HEADER_RESERVED_0_LBN 64
+#define ERF_SC_PACKETISER_HEADER_RESERVED_0_WIDTH 32
+#define ERF_SC_PACKETISER_HEADER_RESERVED_1_LBN 96
+#define ERF_SC_PACKETISER_HEADER_RESERVED_1_WIDTH 32
+#define ERF_SC_PACKETISER_HEADER_RESERVED_2_LBN 128
+#define ERF_SC_PACKETISER_HEADER_RESERVED_2_WIDTH 32
+
+
+/*------------------------------------------------------------*/
+/*
+ * ER_RX_SL_PACKETISER_PAYLOAD_WORD(128bit):
+ *
+ */
+#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE 16
+#define ER_RX_SL_PACKETISER_PAYLOAD_WORD_WIDTH 128
+
+#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN 0
+#define ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH 24
+#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_LBN 24
+#define ERF_SC_PACKETISER_PAYLOAD_RESERVED_WIDTH 8
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_OFST 4
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_SIZE 6
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN 32
+#define ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH 48
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_OFST 10
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_SIZE 6
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN 80
+#define ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH 48
+
+
+#endif /* EFX_MAE_COUNTER_FORMAT_H */
diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h
index 1f18e9dc62e8..7e35fec9da35 100644
--- a/drivers/net/ethernet/sfc/mcdi.h
+++ b/drivers/net/ethernet/sfc/mcdi.h
@@ -221,15 +221,32 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
#define MCDI_BYTE(_buf, _field) \
((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1), \
*MCDI_PTR(_buf, _field))
+#define MCDI_SET_WORD(_buf, _field, _value) do { \
+ BUILD_BUG_ON(MC_CMD_ ## _field ## _LEN != 2); \
+ BUILD_BUG_ON(MC_CMD_ ## _field ## _OFST & 1); \
+ *(__force __le16 *)MCDI_PTR(_buf, _field) = cpu_to_le16(_value);\
+ } while (0)
#define MCDI_WORD(_buf, _field) \
((u16)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 2) + \
le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field)))
+/* Write a 16-bit field defined in the protocol as being big-endian. */
+#define MCDI_STRUCT_SET_WORD_BE(_buf, _field, _value) do { \
+ BUILD_BUG_ON(_field ## _LEN != 2); \
+ BUILD_BUG_ON(_field ## _OFST & 1); \
+ *(__force __be16 *)MCDI_STRUCT_PTR(_buf, _field) = (_value); \
+ } while (0)
#define MCDI_SET_DWORD(_buf, _field, _value) \
EFX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0, _value)
#define MCDI_STRUCT_SET_DWORD(_buf, _field, _value) \
EFX_POPULATE_DWORD_1(*_MCDI_STRUCT_DWORD(_buf, _field), EFX_DWORD_0, _value)
#define MCDI_DWORD(_buf, _field) \
EFX_DWORD_FIELD(*_MCDI_DWORD(_buf, _field), EFX_DWORD_0)
+/* Write a 32-bit field defined in the protocol as being big-endian. */
+#define MCDI_STRUCT_SET_DWORD_BE(_buf, _field, _value) do { \
+ BUILD_BUG_ON(_field ## _LEN != 4); \
+ BUILD_BUG_ON(_field ## _OFST & 3); \
+ *(__force __be32 *)MCDI_STRUCT_PTR(_buf, _field) = (_value); \
+ } while (0)
#define MCDI_POPULATE_DWORD_1(_buf, _field, _name1, _value1) \
EFX_POPULATE_DWORD_1(*_MCDI_DWORD(_buf, _field), \
MC_CMD_ ## _name1, _value1)
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 2e9ba0cfe848..3b49e216768b 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -56,7 +56,8 @@
#define EFX_MAX_RX_QUEUES EFX_MAX_CHANNELS
#define EFX_EXTRA_CHANNEL_IOV 0
#define EFX_EXTRA_CHANNEL_PTP 1
-#define EFX_MAX_EXTRA_CHANNELS 2U
+#define EFX_EXTRA_CHANNEL_TC 2
+#define EFX_MAX_EXTRA_CHANNELS 3U
/* Checksum generation is a per-queue option in hardware, so each
* queue visible to the networking core is backed by two hardware TX
@@ -363,8 +364,12 @@ struct efx_rx_page_state {
* @refill_enabled: Enable refill whenever fill level is low
* @flush_pending: Set when a RX flush is pending. Has the same lifetime as
* @rxq_flush_pending.
+ * @grant_credits: Posted RX descriptors need to be granted to the MAE with
+ * %MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS. For %EFX_EXTRA_CHANNEL_TC,
+ * and only supported on EF100.
* @added_count: Number of buffers added to the receive queue.
* @notified_count: Number of buffers given to NIC (<= @added_count).
+ * @granted_count: Number of buffers granted to the MAE (<= @notified_count).
* @removed_count: Number of buffers removed from the receive queue.
* @scatter_n: Used by NIC specific receive code.
* @scatter_len: Used by NIC specific receive code.
@@ -385,6 +390,7 @@ struct efx_rx_page_state {
* refill was triggered.
* @recycle_count: RX buffer recycle counter.
* @slow_fill: Timer used to defer efx_nic_generate_fill_event().
+ * @grant_work: workitem used to grant credits to the MAE if @grant_credits
* @xdp_rxq_info: XDP specific RX queue information.
* @xdp_rxq_info_valid: Is xdp_rxq_info valid data?.
*/
@@ -396,9 +402,11 @@ struct efx_rx_queue {
unsigned int ptr_mask;
bool refill_enabled;
bool flush_pending;
+ bool grant_credits;
unsigned int added_count;
unsigned int notified_count;
+ unsigned int granted_count;
unsigned int removed_count;
unsigned int scatter_n;
unsigned int scatter_len;
@@ -416,6 +424,7 @@ struct efx_rx_queue {
unsigned int recycle_count;
struct timer_list slow_fill;
unsigned int slow_fill_count;
+ struct work_struct grant_work;
/* Statistics to supplement MAC stats */
unsigned long rx_packets;
struct xdp_rxq_info xdp_rxq_info;
@@ -577,12 +586,15 @@ struct efx_msi_context {
* struct efx_channel_type - distinguishes traffic and extra channels
* @handle_no_channel: Handle failure to allocate an extra channel
* @pre_probe: Set up extra state prior to initialisation
+ * @start: called early in efx_start_channels()
+ * @stop: called early in efx_stop_channels()
* @post_remove: Tear down extra state after finalisation, if allocated.
* May be called on channels that have not been probed.
* @get_name: Generate the channel's name (used for its IRQ handler)
* @copy: Copy the channel state prior to reallocation. May be %NULL if
* reallocation is not supported.
* @receive_skb: Handle an skb ready to be passed to netif_receive_skb()
+ * @receive_raw: Handle an RX buffer ready to be passed to __efx_rx_packet()
* @want_txqs: Determine whether this channel should have TX queues
* created. If %NULL, TX queues are not created.
* @keep_eventq: Flag for whether event queue should be kept initialised
@@ -593,10 +605,13 @@ struct efx_msi_context {
struct efx_channel_type {
void (*handle_no_channel)(struct efx_nic *);
int (*pre_probe)(struct efx_channel *);
+ int (*start)(struct efx_channel *);
+ void (*stop)(struct efx_channel *);
void (*post_remove)(struct efx_channel *);
void (*get_name)(struct efx_channel *, char *buf, size_t len);
struct efx_channel *(*copy)(const struct efx_channel *);
bool (*receive_skb)(struct efx_channel *, struct sk_buff *);
+ bool (*receive_raw)(struct efx_rx_queue *, u32);
bool (*want_txqs)(struct efx_channel *);
bool keep_eventq;
bool want_pio;
@@ -855,7 +870,6 @@ enum efx_xdp_tx_queues_mode {
* @timer_max_ns: Interrupt timer maximum value, in nanoseconds
* @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
* @irqs_hooked: Channel interrupts are hooked
- * @log_tc_errs: Error logging for TC filter insertion is enabled
* @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
* @irq_rx_moderation_us: IRQ moderation time for RX event queues
* @msg_enable: Log message enable flags
@@ -1018,7 +1032,6 @@ struct efx_nic {
unsigned int timer_max_ns;
bool irq_rx_adaptive;
bool irqs_hooked;
- bool log_tc_errs;
unsigned int irq_mod_step_us;
unsigned int irq_rx_moderation_us;
u32 msg_enable;
diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index eaef4a15008a..9f07e1ba7780 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -351,7 +351,7 @@ struct efx_ptp_data {
void (*xmit_skb)(struct efx_nic *efx, struct sk_buff *skb);
};
-static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta);
+static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm);
static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta);
static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts);
static int efx_phc_settime(struct ptp_clock_info *ptp,
@@ -1508,7 +1508,7 @@ static const struct ptp_clock_info efx_phc_clock_info = {
.n_per_out = 0,
.n_pins = 0,
.pps = 1,
- .adjfreq = efx_phc_adjfreq,
+ .adjfine = efx_phc_adjfine,
.adjtime = efx_phc_adjtime,
.gettime64 = efx_phc_gettime,
.settime64 = efx_phc_settime,
@@ -2137,11 +2137,12 @@ void __efx_rx_skb_attach_timestamp(struct efx_channel *channel,
ptp->ts_corrections.general_rx);
}
-static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)
+static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
{
struct efx_ptp_data *ptp_data = container_of(ptp,
struct efx_ptp_data,
phc_clock_info);
+ s32 delta = scaled_ppm_to_ppb(scaled_ppm);
struct efx_nic *efx = ptp_data->efx;
MCDI_DECLARE_BUF(inadj, MC_CMD_PTP_IN_ADJUST_LEN);
s64 adjustment_ns;
diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c
index 9220afeddee8..d2f35ee15eff 100644
--- a/drivers/net/ethernet/sfc/rx_common.c
+++ b/drivers/net/ethernet/sfc/rx_common.c
@@ -229,6 +229,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
/* Initialise ptr fields */
rx_queue->added_count = 0;
rx_queue->notified_count = 0;
+ rx_queue->granted_count = 0;
rx_queue->removed_count = 0;
rx_queue->min_fill = -1U;
efx_init_rx_recycle_ring(rx_queue);
@@ -281,6 +282,8 @@ void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
"shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
del_timer_sync(&rx_queue->slow_fill);
+ if (rx_queue->grant_credits)
+ flush_work(&rx_queue->grant_work);
/* Release RX buffers from the current read ptr to the write ptr */
if (rx_queue->buffer) {
diff --git a/drivers/net/ethernet/sfc/siena/ptp.c b/drivers/net/ethernet/sfc/siena/ptp.c
index 7c46752e6eae..38e666561bcd 100644
--- a/drivers/net/ethernet/sfc/siena/ptp.c
+++ b/drivers/net/ethernet/sfc/siena/ptp.c
@@ -347,7 +347,7 @@ struct efx_ptp_data {
void (*xmit_skb)(struct efx_nic *efx, struct sk_buff *skb);
};
-static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta);
+static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm);
static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta);
static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts);
static int efx_phc_settime(struct ptp_clock_info *ptp,
@@ -1429,7 +1429,7 @@ static const struct ptp_clock_info efx_phc_clock_info = {
.n_per_out = 0,
.n_pins = 0,
.pps = 1,
- .adjfreq = efx_phc_adjfreq,
+ .adjfine = efx_phc_adjfine,
.adjtime = efx_phc_adjtime,
.gettime64 = efx_phc_gettime,
.settime64 = efx_phc_settime,
@@ -2044,11 +2044,12 @@ void __efx_siena_rx_skb_attach_timestamp(struct efx_channel *channel,
ptp->ts_corrections.general_rx);
}
-static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)
+static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
{
struct efx_ptp_data *ptp_data = container_of(ptp,
struct efx_ptp_data,
phc_clock_info);
+ s32 delta = scaled_ppm_to_ppb(scaled_ppm);
struct efx_nic *efx = ptp_data->efx;
MCDI_DECLARE_BUF(inadj, MC_CMD_PTP_IN_ADJUST_LEN);
s64 adjustment_ns;
diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c
index 3478860d4023..deeaab9ee761 100644
--- a/drivers/net/ethernet/sfc/tc.c
+++ b/drivers/net/ethernet/sfc/tc.c
@@ -77,6 +77,8 @@ static void efx_tc_free_action_set(struct efx_nic *efx,
*/
list_del(&act->list);
}
+ if (act->count)
+ efx_tc_flower_put_counter_index(efx, act->count);
kfree(act);
}
@@ -124,50 +126,187 @@ static void efx_tc_flow_free(void *ptr, void *arg)
kfree(rule);
}
+/* Boilerplate for the simple 'copy a field' cases */
+#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
+if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \
+ struct flow_match_##_type fm; \
+ \
+ flow_rule_match_##_tcget(rule, &fm); \
+ match->value._field = fm.key->_tcfield; \
+ match->mask._field = fm.mask->_tcfield; \
+}
+#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \
+ _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
+#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
+ _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
+
static int efx_tc_flower_parse_match(struct efx_nic *efx,
struct flow_rule *rule,
struct efx_tc_match *match,
struct netlink_ext_ack *extack)
{
struct flow_dissector *dissector = rule->match.dissector;
+ unsigned char ipv = 0;
+ /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
+ * even on IPv4 filters; so rather than relying on dissector->used_keys
+ * we check the addr_type in the CONTROL key. If we don't find it (or
+ * it's masked, which should never happen), we treat both IPV4_ADDRS
+ * and IPV6_ADDRS as absent.
+ */
if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
struct flow_match_control fm;
flow_rule_match_control(rule, &fm);
+ if (IS_ALL_ONES(fm.mask->addr_type))
+ switch (fm.key->addr_type) {
+ case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
+ ipv = 4;
+ break;
+ case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
+ ipv = 6;
+ break;
+ default:
+ break;
+ }
- if (fm.mask->flags) {
- efx_tc_err(efx, "Unsupported match on control.flags %#x\n",
- fm.mask->flags);
- NL_SET_ERR_MSG_MOD(extack, "Unsupported match on control.flags");
+ if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
+ match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
+ match->mask.ip_frag = true;
+ }
+ if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
+ match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
+ match->mask.ip_firstfrag = true;
+ }
+ if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x",
+ fm.mask->flags);
return -EOPNOTSUPP;
}
}
if (dissector->used_keys &
~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
- BIT(FLOW_DISSECTOR_KEY_BASIC))) {
- efx_tc_err(efx, "Unsupported flower keys %#x\n", dissector->used_keys);
- NL_SET_ERR_MSG_MOD(extack, "Unsupported flower keys encountered");
+ BIT(FLOW_DISSECTOR_KEY_BASIC) |
+ BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
+ BIT(FLOW_DISSECTOR_KEY_VLAN) |
+ BIT(FLOW_DISSECTOR_KEY_CVLAN) |
+ BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
+ BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
+ BIT(FLOW_DISSECTOR_KEY_PORTS) |
+ BIT(FLOW_DISSECTOR_KEY_TCP) |
+ BIT(FLOW_DISSECTOR_KEY_IP))) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#x",
+ dissector->used_keys);
return -EOPNOTSUPP;
}
- if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
- struct flow_match_basic fm;
+ MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
+ /* Make sure we're IP if any L3/L4 keys used. */
+ if (!IS_ALL_ONES(match->mask.eth_proto) ||
+ !(match->value.eth_proto == htons(ETH_P_IP) ||
+ match->value.eth_proto == htons(ETH_P_IPV6)))
+ if (dissector->used_keys &
+ (BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
+ BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
+ BIT(FLOW_DISSECTOR_KEY_PORTS) |
+ BIT(FLOW_DISSECTOR_KEY_IP) |
+ BIT(FLOW_DISSECTOR_KEY_TCP))) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "L3/L4 flower keys %#x require protocol ipv[46]",
+ dissector->used_keys);
+ return -EINVAL;
+ }
- flow_rule_match_basic(rule, &fm);
- if (fm.mask->n_proto) {
- EFX_TC_ERR_MSG(efx, extack, "Unsupported eth_proto match\n");
- return -EOPNOTSUPP;
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
+ struct flow_match_vlan fm;
+
+ flow_rule_match_vlan(rule, &fm);
+ if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
+ match->value.vlan_proto[0] = fm.key->vlan_tpid;
+ match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
+ match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
+ fm.key->vlan_id);
+ match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
+ fm.mask->vlan_id);
}
- if (fm.mask->ip_proto) {
- EFX_TC_ERR_MSG(efx, extack, "Unsupported ip_proto match\n");
- return -EOPNOTSUPP;
+ }
+
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
+ struct flow_match_vlan fm;
+
+ flow_rule_match_cvlan(rule, &fm);
+ if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
+ match->value.vlan_proto[1] = fm.key->vlan_tpid;
+ match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
+ match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
+ fm.key->vlan_id);
+ match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
+ fm.mask->vlan_id);
}
}
+ if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
+ struct flow_match_eth_addrs fm;
+
+ flow_rule_match_eth_addrs(rule, &fm);
+ ether_addr_copy(match->value.eth_saddr, fm.key->src);
+ ether_addr_copy(match->value.eth_daddr, fm.key->dst);
+ ether_addr_copy(match->mask.eth_saddr, fm.mask->src);
+ ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);
+ }
+
+ MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
+ /* Make sure we're TCP/UDP if any L4 keys used. */
+ if ((match->value.ip_proto != IPPROTO_UDP &&
+ match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
+ if (dissector->used_keys &
+ (BIT(FLOW_DISSECTOR_KEY_PORTS) |
+ BIT(FLOW_DISSECTOR_KEY_TCP))) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "L4 flower keys %#x require ipproto udp or tcp",
+ dissector->used_keys);
+ return -EINVAL;
+ }
+ MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
+ MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
+ if (ipv == 4) {
+ MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
+ MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
+ }
+#ifdef CONFIG_IPV6
+ else if (ipv == 6) {
+ MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
+ MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
+ }
+#endif
+ MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
+ MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
+ MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
+
return 0;
}
+/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
+enum efx_tc_action_order {
+ EFX_TC_AO_COUNT,
+ EFX_TC_AO_DELIVER
+};
+/* Determine whether we can add @new action without violating order */
+static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
+ enum efx_tc_action_order new)
+{
+ switch (new) {
+ case EFX_TC_AO_COUNT:
+ if (act->count)
+ return false;
+ fallthrough;
+ case EFX_TC_AO_DELIVER:
+ return !act->deliver;
+ default:
+ /* Bad caller. Whatever they wanted to do, say they can't. */
+ WARN_ON_ONCE(1);
+ return false;
+ }
+}
+
static int efx_tc_flower_replace(struct efx_nic *efx,
struct net_device *net_dev,
struct flow_cls_offload *tc,
@@ -200,13 +339,9 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
if (efv != from_efv) {
/* can't happen */
- efx_tc_err(efx, "for %s efv is %snull but from_efv is %snull\n",
- netdev_name(net_dev), efv ? "non-" : "",
- from_efv ? "non-" : "");
- if (efv)
- NL_SET_ERR_MSG_MOD(extack, "vfrep filter has PF net_dev (can't happen)");
- else
- NL_SET_ERR_MSG_MOD(extack, "PF filter has vfrep net_dev (can't happen)");
+ NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
+ netdev_name(net_dev), efv ? "non-" : "",
+ from_efv ? "non-" : "");
return -EINVAL;
}
@@ -214,7 +349,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
memset(&match, 0, sizeof(match));
rc = efx_tc_flower_external_mport(efx, from_efv);
if (rc < 0) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to identify ingress m-port");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
return rc;
}
match.value.ingress_port = rc;
@@ -224,7 +359,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
return rc;
if (tc->common.chain_index) {
- EFX_TC_ERR_MSG(efx, extack, "No support for nonzero chain_index");
+ NL_SET_ERR_MSG_MOD(extack, "No support for nonzero chain_index");
return -EOPNOTSUPP;
}
match.mask.recirc_id = 0xff;
@@ -261,16 +396,57 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
if (!act) {
/* more actions after a non-pipe action */
- EFX_TC_ERR_MSG(efx, extack, "Action follows non-pipe action");
+ NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
rc = -EINVAL;
goto release;
}
+ if ((fa->id == FLOW_ACTION_REDIRECT ||
+ fa->id == FLOW_ACTION_MIRRED ||
+ fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
+ struct efx_tc_counter_index *ctr;
+
+ /* Currently the only actions that want stats are
+ * mirred and gact (ok, shot, trap, goto-chain), which
+ * means we want stats just before delivery. Also,
+ * note that tunnel_key set shouldn't change the length
+ * — it's only the subsequent mirred that does that,
+ * and the stats are taken _before_ the mirred action
+ * happens.
+ */
+ if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
+ /* All supported actions that count either steal
+ * (gact shot, mirred redirect) or clone act
+ * (mirred mirror), so we should never get two
+ * count actions on one action_set.
+ */
+ NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
+ rc = -EOPNOTSUPP;
+ goto release;
+ }
+
+ if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
+ NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
+ fa->hw_stats);
+ rc = -EOPNOTSUPP;
+ goto release;
+ }
+
+ ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
+ EFX_TC_COUNTER_TYPE_AR);
+ if (IS_ERR(ctr)) {
+ rc = PTR_ERR(ctr);
+ NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
+ goto release;
+ }
+ act->count = ctr;
+ }
+
switch (fa->id) {
case FLOW_ACTION_DROP:
rc = efx_mae_alloc_action_set(efx, act);
if (rc) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (drop)");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
goto release;
}
list_add_tail(&act->list, &rule->acts.list);
@@ -279,22 +455,30 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
case FLOW_ACTION_REDIRECT:
case FLOW_ACTION_MIRRED:
save = *act;
+
+ if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
+ /* can't happen */
+ rc = -EOPNOTSUPP;
+ NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
+ goto release;
+ }
+
to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
if (IS_ERR(to_efv)) {
- EFX_TC_ERR_MSG(efx, extack, "Mirred egress device not on switch");
+ NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
rc = PTR_ERR(to_efv);
goto release;
}
rc = efx_tc_flower_external_mport(efx, to_efv);
if (rc < 0) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to identify egress m-port");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
goto release;
}
act->dest_mport = rc;
act->deliver = 1;
rc = efx_mae_alloc_action_set(efx, act);
if (rc) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (mirred)");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
goto release;
}
list_add_tail(&act->list, &rule->acts.list);
@@ -302,6 +486,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
if (fa->id == FLOW_ACTION_REDIRECT)
break; /* end of the line */
/* Mirror, so continue on with saved act */
+ save.count = NULL;
act = kzalloc(sizeof(*act), GFP_USER);
if (!act) {
rc = -ENOMEM;
@@ -310,9 +495,9 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
*act = save;
break;
default:
- efx_tc_err(efx, "Unhandled action %u\n", fa->id);
+ NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
+ fa->id);
rc = -EOPNOTSUPP;
- NL_SET_ERR_MSG_MOD(extack, "Unsupported action");
goto release;
}
}
@@ -334,7 +519,7 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
act->deliver = 1;
rc = efx_mae_alloc_action_set(efx, act);
if (rc) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (deliver)");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
goto release;
}
list_add_tail(&act->list, &rule->acts.list);
@@ -349,13 +534,13 @@ static int efx_tc_flower_replace(struct efx_nic *efx,
rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
if (rc) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to write action set list to hw");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
goto release;
}
rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
rule->acts.fw_id, &rule->fw_id);
if (rc) {
- EFX_TC_ERR_MSG(efx, extack, "Failed to insert rule in hw");
+ NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
goto release_acts;
}
return 0;
@@ -410,6 +595,42 @@ static int efx_tc_flower_destroy(struct efx_nic *efx,
return 0;
}
+static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
+ struct flow_cls_offload *tc)
+{
+ struct netlink_ext_ack *extack = tc->common.extack;
+ struct efx_tc_counter_index *ctr;
+ struct efx_tc_counter *cnt;
+ u64 packets, bytes;
+
+ ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
+ if (!ctr) {
+ /* See comment in efx_tc_flower_destroy() */
+ if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
+ if (net_ratelimit())
+ netif_warn(efx, drv, efx->net_dev,
+ "Filter %lx not found for stats\n",
+ tc->cookie);
+ NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
+ return -ENOENT;
+ }
+ if (WARN_ON(!ctr->cnt)) /* can't happen */
+ return -EIO;
+ cnt = ctr->cnt;
+
+ spin_lock_bh(&cnt->lock);
+ /* Report only new pkts/bytes since last time TC asked */
+ packets = cnt->packets;
+ bytes = cnt->bytes;
+ flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
+ packets - cnt->old_packets, 0, cnt->touched,
+ FLOW_ACTION_HW_STATS_DELAYED);
+ cnt->old_packets = packets;
+ cnt->old_bytes = bytes;
+ spin_unlock_bh(&cnt->lock);
+ return 0;
+}
+
int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
struct flow_cls_offload *tc, struct efx_rep *efv)
{
@@ -426,6 +647,9 @@ int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
case FLOW_CLS_DESTROY:
rc = efx_tc_flower_destroy(efx, net_dev, tc);
break;
+ case FLOW_CLS_STATS:
+ rc = efx_tc_flower_stats(efx, net_dev, tc);
+ break;
default:
rc = -EOPNOTSUPP;
break;
@@ -641,6 +865,10 @@ int efx_init_struct_tc(struct efx_nic *efx)
INIT_LIST_HEAD(&efx->tc->block_list);
mutex_init(&efx->tc->mutex);
+ init_waitqueue_head(&efx->tc->flush_wq);
+ rc = efx_tc_init_counters(efx);
+ if (rc < 0)
+ goto fail_counters;
rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
if (rc < 0)
goto fail_match_action_ht;
@@ -650,8 +878,11 @@ int efx_init_struct_tc(struct efx_nic *efx)
efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
+ efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
return 0;
fail_match_action_ht:
+ efx_tc_destroy_counters(efx);
+fail_counters:
mutex_destroy(&efx->tc->mutex);
kfree(efx->tc->caps);
fail_alloc_caps:
@@ -672,6 +903,7 @@ void efx_fini_struct_tc(struct efx_nic *efx)
MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
efx);
+ efx_tc_fini_counters(efx);
mutex_unlock(&efx->tc->mutex);
mutex_destroy(&efx->tc->mutex);
kfree(efx->tc->caps);
diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h
index 196fd74ed973..418ce8c13a06 100644
--- a/drivers/net/ethernet/sfc/tc.h
+++ b/drivers/net/ethernet/sfc/tc.h
@@ -14,27 +14,13 @@
#include <net/flow_offload.h>
#include <linux/rhashtable.h>
#include "net_driver.h"
+#include "tc_counters.h"
-/* Error reporting: convenience macros. For indicating why a given filter
- * insertion is not supported; errors in internal operation or in the
- * hardware should be netif_err()s instead.
- */
-/* Used when error message is constant. */
-#define EFX_TC_ERR_MSG(efx, extack, message) do { \
- NL_SET_ERR_MSG_MOD(extack, message); \
- if (efx->log_tc_errs) \
- netif_info(efx, drv, efx->net_dev, "%s\n", message); \
-} while (0)
-/* Used when error message is not constant; caller should also supply a
- * constant extack message with NL_SET_ERR_MSG_MOD().
- */
-#define efx_tc_err(efx, fmt, args...) do { \
-if (efx->log_tc_errs) \
- netif_info(efx, drv, efx->net_dev, fmt, ##args);\
-} while (0)
+#define IS_ALL_ONES(v) (!(typeof (v))~(v))
struct efx_tc_action_set {
u16 deliver:1;
+ struct efx_tc_counter_index *count;
u32 dest_mport;
u32 fw_id; /* index of this entry in firmware actions table */
struct list_head list;
@@ -44,6 +30,20 @@ struct efx_tc_match_fields {
/* L1 */
u32 ingress_port;
u8 recirc_id;
+ /* L2 (inner when encap) */
+ __be16 eth_proto;
+ __be16 vlan_tci[2], vlan_proto[2];
+ u8 eth_saddr[ETH_ALEN], eth_daddr[ETH_ALEN];
+ /* L3 (when IP) */
+ u8 ip_proto, ip_tos, ip_ttl;
+ __be32 src_ip, dst_ip;
+#ifdef CONFIG_IPV6
+ struct in6_addr src_ip6, dst_ip6;
+#endif
+ bool ip_frag, ip_firstfrag;
+ /* L4 */
+ __be16 l4_sport, l4_dport; /* Ports (UDP, TCP) */
+ __be16 tcp_flags;
};
struct efx_tc_match {
@@ -76,11 +76,19 @@ enum efx_tc_rule_prios {
* @caps: MAE capabilities reported by MCDI
* @block_list: List of &struct efx_tc_block_binding
* @mutex: Used to serialise operations on TC hashtables
+ * @counter_ht: Hashtable of TC counters (FW IDs and counter values)
+ * @counter_id_ht: Hashtable mapping TC counter cookies to counters
* @match_action_ht: Hashtable of TC match-action rules
* @reps_mport_id: MAE port allocated for representor RX
* @reps_filter_uc: VNIC filter for representor unicast RX (promisc)
* @reps_filter_mc: VNIC filter for representor multicast RX (allmulti)
* @reps_mport_vport_id: vport_id for representor RX filters
+ * @flush_counters: counters have been stopped, waiting for drain
+ * @flush_gen: final generation count per type array as reported by
+ * MC_CMD_MAE_COUNTERS_STREAM_STOP
+ * @seen_gen: most recent generation count per type as seen by efx_tc_rx()
+ * @flush_wq: wait queue used by efx_mae_stop_counters() to wait for
+ * MAE counters RXQ to finish draining
* @dflt: Match-action rules for default switching; at priority
* %EFX_TC_PRIO_DFLT. Named by *ingress* port
* @dflt.pf: rule for traffic ingressing from PF (egresses to wire)
@@ -91,9 +99,15 @@ struct efx_tc_state {
struct mae_caps *caps;
struct list_head block_list;
struct mutex mutex;
+ struct rhashtable counter_ht;
+ struct rhashtable counter_id_ht;
struct rhashtable match_action_ht;
u32 reps_mport_id, reps_mport_vport_id;
s32 reps_filter_uc, reps_filter_mc;
+ bool flush_counters;
+ u32 flush_gen[EFX_TC_COUNTER_TYPE_MAX];
+ u32 seen_gen[EFX_TC_COUNTER_TYPE_MAX];
+ wait_queue_head_t flush_wq;
struct {
struct efx_tc_flow_rule pf;
struct efx_tc_flow_rule wire;
diff --git a/drivers/net/ethernet/sfc/tc_counters.c b/drivers/net/ethernet/sfc/tc_counters.c
new file mode 100644
index 000000000000..d1a91d54c6bb
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_counters.c
@@ -0,0 +1,503 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#include "tc_counters.h"
+#include "mae_counter_format.h"
+#include "mae.h"
+#include "rx_common.h"
+
+/* Counter-management hashtables */
+
+static const struct rhashtable_params efx_tc_counter_id_ht_params = {
+ .key_len = offsetof(struct efx_tc_counter_index, linkage),
+ .key_offset = 0,
+ .head_offset = offsetof(struct efx_tc_counter_index, linkage),
+};
+
+static const struct rhashtable_params efx_tc_counter_ht_params = {
+ .key_len = offsetof(struct efx_tc_counter, linkage),
+ .key_offset = 0,
+ .head_offset = offsetof(struct efx_tc_counter, linkage),
+};
+
+static void efx_tc_counter_free(void *ptr, void *__unused)
+{
+ struct efx_tc_counter *cnt = ptr;
+
+ kfree(cnt);
+}
+
+static void efx_tc_counter_id_free(void *ptr, void *__unused)
+{
+ struct efx_tc_counter_index *ctr = ptr;
+
+ WARN_ON(refcount_read(&ctr->ref));
+ kfree(ctr);
+}
+
+int efx_tc_init_counters(struct efx_nic *efx)
+{
+ int rc;
+
+ rc = rhashtable_init(&efx->tc->counter_id_ht, &efx_tc_counter_id_ht_params);
+ if (rc < 0)
+ goto fail_counter_id_ht;
+ rc = rhashtable_init(&efx->tc->counter_ht, &efx_tc_counter_ht_params);
+ if (rc < 0)
+ goto fail_counter_ht;
+ return 0;
+fail_counter_ht:
+ rhashtable_destroy(&efx->tc->counter_id_ht);
+fail_counter_id_ht:
+ return rc;
+}
+
+/* Only call this in init failure teardown.
+ * Normal exit should fini instead as there may be entries in the table.
+ */
+void efx_tc_destroy_counters(struct efx_nic *efx)
+{
+ rhashtable_destroy(&efx->tc->counter_ht);
+ rhashtable_destroy(&efx->tc->counter_id_ht);
+}
+
+void efx_tc_fini_counters(struct efx_nic *efx)
+{
+ rhashtable_free_and_destroy(&efx->tc->counter_id_ht, efx_tc_counter_id_free, NULL);
+ rhashtable_free_and_destroy(&efx->tc->counter_ht, efx_tc_counter_free, NULL);
+}
+
+/* Counter allocation */
+
+static struct efx_tc_counter *efx_tc_flower_allocate_counter(struct efx_nic *efx,
+ int type)
+{
+ struct efx_tc_counter *cnt;
+ int rc, rc2;
+
+ cnt = kzalloc(sizeof(*cnt), GFP_USER);
+ if (!cnt)
+ return ERR_PTR(-ENOMEM);
+
+ spin_lock_init(&cnt->lock);
+ cnt->touched = jiffies;
+ cnt->type = type;
+
+ rc = efx_mae_allocate_counter(efx, cnt);
+ if (rc)
+ goto fail1;
+ rc = rhashtable_insert_fast(&efx->tc->counter_ht, &cnt->linkage,
+ efx_tc_counter_ht_params);
+ if (rc)
+ goto fail2;
+ return cnt;
+fail2:
+ /* If we get here, it implies that we couldn't insert into the table,
+ * which in turn probably means that the fw_id was already taken.
+ * In that case, it's unclear whether we really 'own' the fw_id; but
+ * the firmware seemed to think we did, so it's proper to free it.
+ */
+ rc2 = efx_mae_free_counter(efx, cnt);
+ if (rc2)
+ netif_warn(efx, hw, efx->net_dev,
+ "Failed to free MAE counter %u, rc %d\n",
+ cnt->fw_id, rc2);
+fail1:
+ kfree(cnt);
+ return ERR_PTR(rc > 0 ? -EIO : rc);
+}
+
+static void efx_tc_flower_release_counter(struct efx_nic *efx,
+ struct efx_tc_counter *cnt)
+{
+ int rc;
+
+ rhashtable_remove_fast(&efx->tc->counter_ht, &cnt->linkage,
+ efx_tc_counter_ht_params);
+ rc = efx_mae_free_counter(efx, cnt);
+ if (rc)
+ netif_warn(efx, hw, efx->net_dev,
+ "Failed to free MAE counter %u, rc %d\n",
+ cnt->fw_id, rc);
+ /* This doesn't protect counter updates coming in arbitrarily long
+ * after we deleted the counter. The RCU just ensures that we won't
+ * free the counter while another thread has a pointer to it.
+ * Ensuring we don't update the wrong counter if the ID gets re-used
+ * is handled by the generation count.
+ */
+ synchronize_rcu();
+ EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock));
+ kfree(cnt);
+}
+
+static struct efx_tc_counter *efx_tc_flower_find_counter_by_fw_id(
+ struct efx_nic *efx, int type, u32 fw_id)
+{
+ struct efx_tc_counter key = {};
+
+ key.fw_id = fw_id;
+ key.type = type;
+
+ return rhashtable_lookup_fast(&efx->tc->counter_ht, &key,
+ efx_tc_counter_ht_params);
+}
+
+/* TC cookie to counter mapping */
+
+void efx_tc_flower_put_counter_index(struct efx_nic *efx,
+ struct efx_tc_counter_index *ctr)
+{
+ if (!refcount_dec_and_test(&ctr->ref))
+ return; /* still in use */
+ rhashtable_remove_fast(&efx->tc->counter_id_ht, &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ efx_tc_flower_release_counter(efx, ctr->cnt);
+ kfree(ctr);
+}
+
+struct efx_tc_counter_index *efx_tc_flower_get_counter_index(
+ struct efx_nic *efx, unsigned long cookie,
+ enum efx_tc_counter_type type)
+{
+ struct efx_tc_counter_index *ctr, *old;
+ struct efx_tc_counter *cnt;
+
+ ctr = kzalloc(sizeof(*ctr), GFP_USER);
+ if (!ctr)
+ return ERR_PTR(-ENOMEM);
+ ctr->cookie = cookie;
+ old = rhashtable_lookup_get_insert_fast(&efx->tc->counter_id_ht,
+ &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ if (old) {
+ /* don't need our new entry */
+ kfree(ctr);
+ if (!refcount_inc_not_zero(&old->ref))
+ return ERR_PTR(-EAGAIN);
+ /* existing entry found */
+ ctr = old;
+ } else {
+ cnt = efx_tc_flower_allocate_counter(efx, type);
+ if (IS_ERR(cnt)) {
+ rhashtable_remove_fast(&efx->tc->counter_id_ht,
+ &ctr->linkage,
+ efx_tc_counter_id_ht_params);
+ kfree(ctr);
+ return (void *)cnt; /* it's an ERR_PTR */
+ }
+ ctr->cnt = cnt;
+ refcount_set(&ctr->ref, 1);
+ }
+ return ctr;
+}
+
+struct efx_tc_counter_index *efx_tc_flower_find_counter_index(
+ struct efx_nic *efx, unsigned long cookie)
+{
+ struct efx_tc_counter_index key = {};
+
+ key.cookie = cookie;
+ return rhashtable_lookup_fast(&efx->tc->counter_id_ht, &key,
+ efx_tc_counter_id_ht_params);
+}
+
+/* TC Channel. Counter updates are delivered on this channel's RXQ. */
+
+static void efx_tc_handle_no_channel(struct efx_nic *efx)
+{
+ netif_warn(efx, drv, efx->net_dev,
+ "MAE counters require MSI-X and 1 additional interrupt vector.\n");
+}
+
+static int efx_tc_probe_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = &channel->rx_queue;
+
+ channel->irq_moderation_us = 0;
+ rx_queue->core_index = 0;
+
+ INIT_WORK(&rx_queue->grant_work, efx_mae_counters_grant_credits);
+
+ return 0;
+}
+
+static int efx_tc_start_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
+ struct efx_nic *efx = channel->efx;
+
+ return efx_mae_start_counters(efx, rx_queue);
+}
+
+static void efx_tc_stop_channel(struct efx_channel *channel)
+{
+ struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
+ struct efx_nic *efx = channel->efx;
+ int rc;
+
+ rc = efx_mae_stop_counters(efx, rx_queue);
+ if (rc)
+ netif_warn(efx, drv, efx->net_dev,
+ "Failed to stop MAE counters streaming, rc=%d.\n",
+ rc);
+ rx_queue->grant_credits = false;
+ flush_work(&rx_queue->grant_work);
+}
+
+static void efx_tc_remove_channel(struct efx_channel *channel)
+{
+}
+
+static void efx_tc_get_channel_name(struct efx_channel *channel,
+ char *buf, size_t len)
+{
+ snprintf(buf, len, "%s-mae", channel->efx->name);
+}
+
+static void efx_tc_counter_update(struct efx_nic *efx,
+ enum efx_tc_counter_type counter_type,
+ u32 counter_idx, u64 packets, u64 bytes,
+ u32 mark)
+{
+ struct efx_tc_counter *cnt;
+
+ rcu_read_lock(); /* Protect against deletion of 'cnt' */
+ cnt = efx_tc_flower_find_counter_by_fw_id(efx, counter_type, counter_idx);
+ if (!cnt) {
+ /* This can legitimately happen when a counter is removed,
+ * with updates for the counter still in-flight; however this
+ * should be an infrequent occurrence.
+ */
+ if (net_ratelimit())
+ netif_dbg(efx, drv, efx->net_dev,
+ "Got update for unwanted MAE counter %u type %u\n",
+ counter_idx, counter_type);
+ goto out;
+ }
+
+ spin_lock_bh(&cnt->lock);
+ if ((s32)mark - (s32)cnt->gen < 0) {
+ /* This counter update packet is from before the counter was
+ * allocated; thus it must be for a previous counter with
+ * the same ID that has since been freed, and it should be
+ * ignored.
+ */
+ } else {
+ /* Update latest seen generation count. This ensures that
+ * even a long-lived counter won't start getting ignored if
+ * the generation count wraps around, unless it somehow
+ * manages to go 1<<31 generations without an update.
+ */
+ cnt->gen = mark;
+ /* update counter values */
+ cnt->packets += packets;
+ cnt->bytes += bytes;
+ cnt->touched = jiffies;
+ }
+ spin_unlock_bh(&cnt->lock);
+out:
+ rcu_read_unlock();
+}
+
+static void efx_tc_rx_version_1(struct efx_nic *efx, const u8 *data, u32 mark)
+{
+ u16 n_counters, i;
+
+ /* Header format:
+ * + | 0 | 1 | 2 | 3 |
+ * 0 |version | reserved |
+ * 4 | seq_index | n_counters |
+ */
+
+ n_counters = le16_to_cpu(*(const __le16 *)(data + 6));
+
+ /* Counter update entry format:
+ * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f |
+ * | counter_idx | packet_count | byte_count |
+ */
+ for (i = 0; i < n_counters; i++) {
+ const void *entry = data + 8 + 16 * i;
+ u64 packet_count, byte_count;
+ u32 counter_idx;
+
+ counter_idx = le32_to_cpu(*(const __le32 *)entry);
+ packet_count = le32_to_cpu(*(const __le32 *)(entry + 4)) |
+ ((u64)le16_to_cpu(*(const __le16 *)(entry + 8)) << 32);
+ byte_count = le16_to_cpu(*(const __le16 *)(entry + 10)) |
+ ((u64)le32_to_cpu(*(const __le32 *)(entry + 12)) << 16);
+ efx_tc_counter_update(efx, EFX_TC_COUNTER_TYPE_AR, counter_idx,
+ packet_count, byte_count, mark);
+ }
+}
+
+#define TCV2_HDR_PTR(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 7), \
+ (pkt) + ERF_SC_PACKETISER_HEADER_##field##_LBN / 8)
+#define TCV2_HDR_BYTE(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 8),\
+ *TCV2_HDR_PTR(pkt, field))
+#define TCV2_HDR_WORD(pkt, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 16),\
+ (void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 15), \
+ *(__force const __le16 *)TCV2_HDR_PTR(pkt, field))
+#define TCV2_PKT_PTR(pkt, poff, i, field) \
+ ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_PAYLOAD_##field##_LBN & 7), \
+ (pkt) + ERF_SC_PACKETISER_PAYLOAD_##field##_LBN/8 + poff + \
+ i * ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE)
+
+/* Read a little-endian 48-bit field with 16-bit alignment */
+static u64 efx_tc_read48(const __le16 *field)
+{
+ u64 out = 0;
+ int i;
+
+ for (i = 0; i < 3; i++)
+ out |= (u64)le16_to_cpu(field[i]) << (i * 16);
+ return out;
+}
+
+static enum efx_tc_counter_type efx_tc_rx_version_2(struct efx_nic *efx,
+ const u8 *data, u32 mark)
+{
+ u8 payload_offset, header_offset, ident;
+ enum efx_tc_counter_type type;
+ u16 n_counters, i;
+
+ ident = TCV2_HDR_BYTE(data, IDENTIFIER);
+ switch (ident) {
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR:
+ type = EFX_TC_COUNTER_TYPE_AR;
+ break;
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT:
+ type = EFX_TC_COUNTER_TYPE_CT;
+ break;
+ case ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR:
+ type = EFX_TC_COUNTER_TYPE_OR;
+ break;
+ default:
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "ignored v2 MAE counter packet (bad identifier %u"
+ "), counters may be inaccurate\n", ident);
+ return EFX_TC_COUNTER_TYPE_MAX;
+ }
+ header_offset = TCV2_HDR_BYTE(data, HEADER_OFFSET);
+ /* mae_counter_format.h implies that this offset is fixed, since it
+ * carries on with SOP-based LBNs for the fields in this header
+ */
+ if (header_offset != ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT) {
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "choked on v2 MAE counter packet (bad header_offset %u"
+ "), counters may be inaccurate\n", header_offset);
+ return EFX_TC_COUNTER_TYPE_MAX;
+ }
+ payload_offset = TCV2_HDR_BYTE(data, PAYLOAD_OFFSET);
+ n_counters = le16_to_cpu(TCV2_HDR_WORD(data, COUNT));
+
+ for (i = 0; i < n_counters; i++) {
+ const void *counter_idx_p, *packet_count_p, *byte_count_p;
+ u64 packet_count, byte_count;
+ u32 counter_idx;
+
+ /* 24-bit field with 32-bit alignment */
+ counter_idx_p = TCV2_PKT_PTR(data, payload_offset, i, COUNTER_INDEX);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH != 24);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN & 31);
+ counter_idx = le32_to_cpu(*(const __le32 *)counter_idx_p) & 0xffffff;
+ /* 48-bit field with 16-bit alignment */
+ packet_count_p = TCV2_PKT_PTR(data, payload_offset, i, PACKET_COUNT);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH != 48);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN & 15);
+ packet_count = efx_tc_read48((const __le16 *)packet_count_p);
+ /* 48-bit field with 16-bit alignment */
+ byte_count_p = TCV2_PKT_PTR(data, payload_offset, i, BYTE_COUNT);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH != 48);
+ BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN & 15);
+ byte_count = efx_tc_read48((const __le16 *)byte_count_p);
+
+ if (type == EFX_TC_COUNTER_TYPE_CT) {
+ /* CT counters are 1-bit saturating counters to update
+ * the lastuse time in CT stats. A received CT counter
+ * should have packet counter to 0 and only LSB bit on
+ * in byte counter.
+ */
+ if (packet_count || byte_count != 1)
+ netdev_warn_once(efx->net_dev,
+ "CT counter with inconsistent state (%llu, %llu)\n",
+ packet_count, byte_count);
+ /* Do not increment the driver's byte counter */
+ byte_count = 0;
+ }
+
+ efx_tc_counter_update(efx, type, counter_idx, packet_count,
+ byte_count, mark);
+ }
+ return type;
+}
+
+/* We always swallow the packet, whether successful or not, since it's not
+ * a network packet and shouldn't ever be forwarded to the stack.
+ * @mark is the generation count for counter allocations.
+ */
+static bool efx_tc_rx(struct efx_rx_queue *rx_queue, u32 mark)
+{
+ struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
+ struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue,
+ channel->rx_pkt_index);
+ const u8 *data = efx_rx_buf_va(rx_buf);
+ struct efx_nic *efx = rx_queue->efx;
+ enum efx_tc_counter_type type;
+ u8 version;
+
+ /* version is always first byte of packet */
+ version = *data;
+ switch (version) {
+ case 1:
+ type = EFX_TC_COUNTER_TYPE_AR;
+ efx_tc_rx_version_1(efx, data, mark);
+ break;
+ case ERF_SC_PACKETISER_HEADER_VERSION_VALUE: // 2
+ type = efx_tc_rx_version_2(efx, data, mark);
+ break;
+ default:
+ if (net_ratelimit())
+ netif_err(efx, drv, efx->net_dev,
+ "choked on MAE counter packet (bad version %u"
+ "); counters may be inaccurate\n",
+ version);
+ goto out;
+ }
+
+ if (type < EFX_TC_COUNTER_TYPE_MAX) {
+ /* Update seen_gen unconditionally, to avoid a missed wakeup if
+ * we race with efx_mae_stop_counters().
+ */
+ efx->tc->seen_gen[type] = mark;
+ if (efx->tc->flush_counters &&
+ (s32)(efx->tc->flush_gen[type] - mark) <= 0)
+ wake_up(&efx->tc->flush_wq);
+ }
+out:
+ efx_free_rx_buffers(rx_queue, rx_buf, 1);
+ channel->rx_pkt_n_frags = 0;
+ return true;
+}
+
+const struct efx_channel_type efx_tc_channel_type = {
+ .handle_no_channel = efx_tc_handle_no_channel,
+ .pre_probe = efx_tc_probe_channel,
+ .start = efx_tc_start_channel,
+ .stop = efx_tc_stop_channel,
+ .post_remove = efx_tc_remove_channel,
+ .get_name = efx_tc_get_channel_name,
+ .receive_raw = efx_tc_rx,
+ .keep_eventq = true,
+};
diff --git a/drivers/net/ethernet/sfc/tc_counters.h b/drivers/net/ethernet/sfc/tc_counters.h
new file mode 100644
index 000000000000..8fc7c4bbb29c
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_counters.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#ifndef EFX_TC_COUNTERS_H
+#define EFX_TC_COUNTERS_H
+#include <linux/refcount.h>
+#include "net_driver.h"
+
+#include "mcdi_pcol.h" /* for MAE_COUNTER_TYPE_* */
+
+enum efx_tc_counter_type {
+ EFX_TC_COUNTER_TYPE_AR = MAE_COUNTER_TYPE_AR,
+ EFX_TC_COUNTER_TYPE_CT = MAE_COUNTER_TYPE_CT,
+ EFX_TC_COUNTER_TYPE_OR = MAE_COUNTER_TYPE_OR,
+ EFX_TC_COUNTER_TYPE_MAX
+};
+
+struct efx_tc_counter {
+ u32 fw_id; /* index in firmware counter table */
+ enum efx_tc_counter_type type;
+ struct rhash_head linkage; /* efx->tc->counter_ht */
+ spinlock_t lock; /* Serialises updates to counter values */
+ u32 gen; /* Generation count at which this counter is current */
+ u64 packets, bytes;
+ u64 old_packets, old_bytes; /* Values last time passed to userspace */
+ /* jiffies of the last time we saw packets increase */
+ unsigned long touched;
+};
+
+struct efx_tc_counter_index {
+ unsigned long cookie;
+ struct rhash_head linkage; /* efx->tc->counter_id_ht */
+ refcount_t ref;
+ struct efx_tc_counter *cnt;
+};
+
+/* create/uncreate/teardown hashtables */
+int efx_tc_init_counters(struct efx_nic *efx);
+void efx_tc_destroy_counters(struct efx_nic *efx);
+void efx_tc_fini_counters(struct efx_nic *efx);
+
+struct efx_tc_counter_index *efx_tc_flower_get_counter_index(
+ struct efx_nic *efx, unsigned long cookie,
+ enum efx_tc_counter_type type);
+void efx_tc_flower_put_counter_index(struct efx_nic *efx,
+ struct efx_tc_counter_index *ctr);
+struct efx_tc_counter_index *efx_tc_flower_find_counter_index(
+ struct efx_nic *efx, unsigned long cookie);
+
+extern const struct efx_channel_type efx_tc_channel_type;
+
+#endif /* EFX_TC_COUNTERS_H */
diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
index c5f88f7a7a04..4ed4082836a9 100644
--- a/drivers/net/ethernet/sfc/tx.c
+++ b/drivers/net/ethernet/sfc/tx.c
@@ -207,11 +207,11 @@ static void efx_skb_copy_bits_to_pio(struct efx_nic *efx, struct sk_buff *skb,
skb_frag_t *f = &skb_shinfo(skb)->frags[i];
u8 *vaddr;
- vaddr = kmap_atomic(skb_frag_page(f));
+ vaddr = kmap_local_page(skb_frag_page(f));
efx_memcpy_toio_aligned_cb(efx, piobuf, vaddr + skb_frag_off(f),
skb_frag_size(f), copy_buf);
- kunmap_atomic(vaddr);
+ kunmap_local(vaddr);
}
EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->frag_list);
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-19 10:13:05 +0000