diff options
Diffstat (limited to 'drivers')
9 files changed, 323 insertions, 14 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index cd4a1ab0ea78..06f511fcbd8e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -47,7 +47,7 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \ en/tc_tun_mplsoudp.o diag/en_tc_tracepoint.o \ en/tc/post_act.o en/tc/int_port.o en/tc/meter.o \ - en/tc/post_meter.o + en/tc/post_meter.o en/tc/act_stats.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/act/trap.o \ en/tc/act/accept.o en/tc/act/mark.o en/tc/act/goto.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index b08339d986d5..3b590cfe33b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -589,7 +589,7 @@ mlx5e_rep_indr_stats_act(struct mlx5e_rep_priv *rpriv, act = mlx5e_tc_act_get(fl_act->id, ns_type); if (!act || !act->stats_action) - return -EOPNOTSUPP; + return mlx5e_tc_fill_action_stats(priv, fl_act); return act->stats_action(priv, fl_act); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c new file mode 100644 index 000000000000..f71766dca660 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include <linux/rhashtable.h> +#include <net/flow_offload.h> +#include "en/tc_priv.h" +#include "act_stats.h" +#include "en/fs.h" + +struct mlx5e_tc_act_stats_handle { + struct rhashtable ht; + spinlock_t ht_lock; /* protects hashtable */ +}; + +struct mlx5e_tc_act_stats { + unsigned long tc_act_cookie; + + struct mlx5_fc *counter; + u64 lastpackets; + u64 lastbytes; + + struct rhash_head hash; + struct rcu_head rcu_head; +}; + +static const struct rhashtable_params act_counters_ht_params = { + .head_offset = offsetof(struct mlx5e_tc_act_stats, hash), + .key_offset = 0, + .key_len = offsetof(struct mlx5e_tc_act_stats, counter), + .automatic_shrinking = true, +}; + +struct mlx5e_tc_act_stats_handle * +mlx5e_tc_act_stats_create(void) +{ + struct mlx5e_tc_act_stats_handle *handle; + int err; + + handle = kvzalloc(sizeof(*handle), GFP_KERNEL); + if (IS_ERR(handle)) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(&handle->ht, &act_counters_ht_params); + if (err) + goto err; + + spin_lock_init(&handle->ht_lock); + return handle; +err: + kvfree(handle); + return ERR_PTR(err); +} + +void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle) +{ + rhashtable_destroy(&handle->ht); + kvfree(handle); +} + +static int +mlx5e_tc_act_stats_add(struct mlx5e_tc_act_stats_handle *handle, + unsigned long act_cookie, + struct mlx5_fc *counter) +{ + struct mlx5e_tc_act_stats *act_stats, *old_act_stats; + struct rhashtable *ht = &handle->ht; + int err = 0; + + act_stats = kvzalloc(sizeof(*act_stats), GFP_KERNEL); + if (!act_stats) + return -ENOMEM; + + act_stats->tc_act_cookie = act_cookie; + act_stats->counter = counter; + + rcu_read_lock(); + old_act_stats = rhashtable_lookup_get_insert_fast(ht, + &act_stats->hash, + act_counters_ht_params); + if (IS_ERR(old_act_stats)) { + err = PTR_ERR(old_act_stats); + goto err_hash_insert; + } else if (old_act_stats) { + err = -EEXIST; + goto err_hash_insert; + } + rcu_read_unlock(); + + return 0; + +err_hash_insert: + rcu_read_unlock(); + kvfree(act_stats); + return err; +} + +void +mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow) +{ + struct mlx5_flow_attr *attr; + struct mlx5e_tc_act_stats *act_stats; + int i; + + if (!flow_flag_test(flow, USE_ACT_STATS)) + return; + + list_for_each_entry(attr, &flow->attrs, list) { + for (i = 0; i < attr->tc_act_cookies_count; i++) { + struct rhashtable *ht = &handle->ht; + + spin_lock(&handle->ht_lock); + act_stats = rhashtable_lookup_fast(ht, + &attr->tc_act_cookies[i], + act_counters_ht_params); + if (act_stats && + rhashtable_remove_fast(ht, &act_stats->hash, + act_counters_ht_params) == 0) + kvfree_rcu(act_stats, rcu_head); + + spin_unlock(&handle->ht_lock); + } + } +} + +int +mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow) +{ + struct mlx5_fc *curr_counter = NULL; + unsigned long last_cookie = 0; + struct mlx5_flow_attr *attr; + int err; + int i; + + if (!flow_flag_test(flow, USE_ACT_STATS)) + return 0; + + list_for_each_entry(attr, &flow->attrs, list) { + if (attr->counter) + curr_counter = attr->counter; + + for (i = 0; i < attr->tc_act_cookies_count; i++) { + /* jump over identical ids (e.g. pedit)*/ + if (last_cookie == attr->tc_act_cookies[i]) + continue; + + err = mlx5e_tc_act_stats_add(handle, attr->tc_act_cookies[i], curr_counter); + if (err) + goto out_err; + last_cookie = attr->tc_act_cookies[i]; + } + } + + return 0; +out_err: + mlx5e_tc_act_stats_del_flow(handle, flow); + return err; +} + +int +mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle, + struct flow_offload_action *fl_act) +{ + struct rhashtable *ht = &handle->ht; + struct mlx5e_tc_act_stats *item; + struct mlx5e_tc_act_stats key; + u64 pkts, bytes, lastused; + int err = 0; + + key.tc_act_cookie = fl_act->cookie; + + rcu_read_lock(); + item = rhashtable_lookup(ht, &key, act_counters_ht_params); + if (!item) { + rcu_read_unlock(); + err = -ENOENT; + goto err_out; + } + + mlx5_fc_query_cached_raw(item->counter, + &bytes, &pkts, &lastused); + + flow_stats_update(&fl_act->stats, + bytes - item->lastbytes, + pkts - item->lastpackets, + 0, lastused, FLOW_ACTION_HW_STATS_DELAYED); + + item->lastpackets = pkts; + item->lastbytes = bytes; + rcu_read_unlock(); + + return 0; + +err_out: + return err; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h new file mode 100644 index 000000000000..002292c2567c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act_stats.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_ACT_STATS_H__ +#define __MLX5_EN_ACT_STATS_H__ + +#include <net/flow_offload.h> +#include "en/tc_priv.h" + +struct mlx5e_tc_act_stats_handle; + +struct mlx5e_tc_act_stats_handle *mlx5e_tc_act_stats_create(void); +void mlx5e_tc_act_stats_free(struct mlx5e_tc_act_stats_handle *handle); + +int +mlx5e_tc_act_stats_add_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow); + +void +mlx5e_tc_act_stats_del_flow(struct mlx5e_tc_act_stats_handle *handle, + struct mlx5e_tc_flow *flow); + +int +mlx5e_tc_act_stats_fill_stats(struct mlx5e_tc_act_stats_handle *handle, + struct flow_offload_action *fl_act); + +#endif /* __MLX5_EN_ACT_STATS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index f575646d2f50..451fd4342a5a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -30,6 +30,7 @@ enum { MLX5E_TC_FLOW_FLAG_TUN_RX = MLX5E_TC_FLOW_BASE + 9, MLX5E_TC_FLOW_FLAG_FAILED = MLX5E_TC_FLOW_BASE + 10, MLX5E_TC_FLOW_FLAG_SAMPLE = MLX5E_TC_FLOW_BASE + 11, + MLX5E_TC_FLOW_FLAG_USE_ACT_STATS = MLX5E_TC_FLOW_BASE + 12, }; struct mlx5e_tc_flow_parse_attr { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index b4e691760da9..0abe3313c673 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -100,6 +100,9 @@ struct mlx5_rep_uplink_priv { struct mlx5e_tc_int_port_priv *int_port_priv; struct mlx5e_flow_meters *flow_meters; + + /* tc action stats */ + struct mlx5e_tc_act_stats_handle *action_stats_handle; }; struct mlx5e_rep_priv { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index e2ec80ebde58..2d06b4412762 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -45,6 +45,7 @@ #include <net/bonding.h> #include "en.h" #include "en/tc/post_act.h" +#include "en/tc/act_stats.h" #include "en_rep.h" #include "en/rep/tc.h" #include "en/rep/neigh.h" @@ -101,6 +102,9 @@ struct mlx5e_tc_table { struct mapping_ctx *mapping; struct mlx5e_hairpin_params hairpin_params; struct dentry *dfs_root; + + /* tc action stats */ + struct mlx5e_tc_act_stats_handle *action_stats_handle; }; struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { @@ -286,6 +290,24 @@ mlx5e_tc_match_to_reg_set_and_get_id(struct mlx5_core_dev *mdev, return err; } +static struct mlx5e_tc_act_stats_handle * +get_act_stats_handle(struct mlx5e_priv *priv) +{ + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + struct mlx5_rep_uplink_priv *uplink_priv; + struct mlx5e_rep_priv *uplink_rpriv; + + if (is_mdev_switchdev_mode(priv->mdev)) { + uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); + uplink_priv = &uplink_rpriv->uplink_priv; + + return uplink_priv->action_stats_handle; + } + + return tc->action_stats_handle; +} + struct mlx5e_tc_int_port_priv * mlx5e_get_int_port_priv(struct mlx5e_priv *priv) { @@ -2026,6 +2048,10 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_out; + err = mlx5e_tc_act_stats_add_flow(get_act_stats_handle(priv), flow); + if (err) + goto err_out; + /* we get here if one of the following takes place: * (1) there's no error * (2) there's an encap action and we don't have valid neigh @@ -2120,6 +2146,8 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, if (flow_flag_test(flow, L3_TO_L2_DECAP)) mlx5e_detach_decap(priv, flow); + mlx5e_tc_act_stats_del_flow(get_act_stats_handle(priv), flow); + free_flow_post_acts(flow); free_branch_attr(flow, attr->branch_true); free_branch_attr(flow, attr->branch_false); @@ -3785,6 +3813,8 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr, INIT_LIST_HEAD(&attr2->list); parse_attr->filter_dev = attr->parse_attr->filter_dev; attr2->action = 0; + attr2->counter = NULL; + attr->tc_act_cookies_count = 0; attr2->flags = 0; attr2->parse_attr = parse_attr; attr2->dest_chain = 0; @@ -4084,6 +4114,11 @@ parse_branch_ctrl(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act, jump_state->jumping_attr = attr->branch_false; jump_state->jump_count = jump_count; + + /* branching action requires its own counter */ + attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + flow_flag_set(flow, USE_ACT_STATS); + return 0; err_branch_false: @@ -4144,6 +4179,8 @@ parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state, goto out_free; parse_state->actions |= attr->action; + if (!tc_act->stats_action) + attr->tc_act_cookies[attr->tc_act_cookies_count++] = act->act_cookie; /* Split attr for multi table act if not the last act. */ if (jump_state.jump_target || @@ -4931,6 +4968,12 @@ errout: return err; } +int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act) +{ + return mlx5e_tc_act_stats_fill_stats(get_act_stats_handle(priv), fl_act); +} + int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags) { @@ -4957,11 +5000,15 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, } if (mlx5e_is_offloaded_flow(flow) || flow_flag_test(flow, CT)) { - counter = mlx5e_tc_get_counter(flow); - if (!counter) - goto errout; + if (flow_flag_test(flow, USE_ACT_STATS)) { + f->use_act_stats = true; + } else { + counter = mlx5e_tc_get_counter(flow); + if (!counter) + goto errout; - mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); + mlx5_fc_query_cached(counter, &bytes, &packets, &lastuse); + } } /* Under multipath it's possible for one rule to be currently @@ -4977,14 +5024,18 @@ int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, u64 packets2; u64 lastuse2; - counter = mlx5e_tc_get_counter(flow->peer_flow); - if (!counter) - goto no_peer_counter; - mlx5_fc_query_cached(counter, &bytes2, &packets2, &lastuse2); - - bytes += bytes2; - packets += packets2; - lastuse = max_t(u64, lastuse, lastuse2); + if (flow_flag_test(flow, USE_ACT_STATS)) { + f->use_act_stats = true; + } else { + counter = mlx5e_tc_get_counter(flow->peer_flow); + if (!counter) + goto no_peer_counter; + mlx5_fc_query_cached(counter, &bytes2, &packets2, &lastuse2); + + bytes += bytes2; + packets += packets2; + lastuse = max_t(u64, lastuse, lastuse2); + } } no_peer_counter: @@ -5323,8 +5374,16 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) mlx5e_tc_debugfs_init(tc, mlx5e_fs_get_debugfs_root(priv->fs)); + tc->action_stats_handle = mlx5e_tc_act_stats_create(); + if (IS_ERR(tc->action_stats_handle)) + goto err_act_stats; + return 0; +err_act_stats: + unregister_netdevice_notifier_dev_net(priv->netdev, + &tc->netdevice_nb, + &tc->netdevice_nn); err_reg: mlx5_tc_ct_clean(tc->ct); mlx5e_tc_post_act_destroy(tc->post_act); @@ -5374,6 +5433,7 @@ void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) mapping_destroy(tc->mapping); mlx5_chains_destroy(tc->chains); mlx5e_tc_nic_destroy_miss_table(priv); + mlx5e_tc_act_stats_free(tc->action_stats_handle); } int mlx5e_tc_ht_init(struct rhashtable *tc_ht) @@ -5450,8 +5510,14 @@ int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv) goto err_register_fib_notifier; } + uplink_priv->action_stats_handle = mlx5e_tc_act_stats_create(); + if (IS_ERR(uplink_priv->action_stats_handle)) + goto err_action_counter; + return 0; +err_action_counter: + mlx5e_tc_tun_cleanup(uplink_priv->encap); err_register_fib_notifier: mapping_destroy(uplink_priv->tunnel_enc_opts_mapping); err_enc_opts_mapping: @@ -5478,6 +5544,7 @@ void mlx5e_tc_esw_cleanup(struct mlx5_rep_uplink_priv *uplink_priv) mlx5_tc_ct_clean(uplink_priv->ct_priv); mlx5e_flow_meters_cleanup(uplink_priv->flow_meters); mlx5e_tc_post_act_destroy(uplink_priv->post_act); + mlx5e_tc_act_stats_free(uplink_priv->action_stats_handle); } int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index ce516dc7f3fd..e8e39fdcda73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -69,6 +69,7 @@ struct mlx5_nic_flow_attr { struct mlx5_flow_attr { u32 action; + unsigned long tc_act_cookies[TCA_ACT_MAX_PRIO]; struct mlx5_fc *counter; struct mlx5_modify_hdr *modify_hdr; struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */ @@ -79,6 +80,7 @@ struct mlx5_flow_attr { struct mlx5e_tc_flow_parse_attr *parse_attr; u32 chain; u16 prio; + u16 tc_act_cookies_count; u32 dest_chain; struct mlx5_flow_table *ft; struct mlx5_flow_table *dest_ft; @@ -197,6 +199,8 @@ int mlx5e_delete_flower(struct net_device *dev, struct mlx5e_priv *priv, int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv, struct flow_cls_offload *f, unsigned long flags); +int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act); int mlx5e_tc_configure_matchall(struct mlx5e_priv *priv, struct tc_cls_matchall_offload *f); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c index b406e0367af6..17fe30a4c06c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c @@ -504,6 +504,16 @@ void mlx5_fc_query_cached(struct mlx5_fc *counter, counter->lastpackets = c.packets; } +void mlx5_fc_query_cached_raw(struct mlx5_fc *counter, + u64 *bytes, u64 *packets, u64 *lastuse) +{ + struct mlx5_fc_cache c = counter->cache; + + *bytes = c.bytes; + *packets = c.packets; + *lastuse = c.lastuse; +} + void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, struct delayed_work *dwork, unsigned long delay) |