From 042637019ea3661bfb82aed6860f73bdbcb6d263 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 10 Jan 2022 14:34:19 +0200 Subject: [PATCH 01/15] net/mlx4: Delete useless moduleparam include Remove inclusion of not used moduleparam.h. Signed-off-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 817f4154b86d..f777151d226f 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include "mlx4_en.h" From 71ab580705c1c49a3e939997d19d1e439cecdb80 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 11 Jan 2022 10:37:52 +0200 Subject: [PATCH 02/15] net/mlx5: Delete useless module.h include There is no need in include of module.h in the following files. Signed-off-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/cq.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/debugfs.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/eq.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/fpga/core.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/fw.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/health.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/mcg.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/mr.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/pd.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/rl.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/uar.c | 1 - 18 files changed, 18 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 989e7cb48a95..c2462d37f1b3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cq.c b/drivers/net/ethernet/mellanox/mlx5/core/cq.c index 15a74966be7d..4caa1b6f40ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cq.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index d69bac93a83b..3d3e55a5cb11 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 7cab08a2f715..299e3f0fcb5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -35,7 +35,6 @@ #include #include #include -#include #include "en.h" #include "en_accel/ipsec.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 48a45aa54a3c..cb4730cb456a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -5,7 +5,6 @@ #include #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.c index 2ce4241459ce..39c03dcbd196 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/core.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 2d8406fab844..614687e0e3d9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -32,7 +32,6 @@ #include #include -#include #include "mlx5_core.h" #include "../../mlxfw/mlxfw.h" #include "lib/tout.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 737df402c927..659021c31cbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c index e042e0924079..4571c56ec3c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/port_tun.c @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ /* Copyright (c) 2019 Mellanox Technologies. */ -#include #include #include #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c index e3b0a131c3e1..d55e15c1f380 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/vxlan.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mcg.c b/drivers/net/ethernet/mellanox/mlx5/core/mcg.c index e019d68062d8..495cca58dccc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mcg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/mcg.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c index f099a087400e..9d735c343a3b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c @@ -31,7 +31,6 @@ */ #include -#include #include #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index e0543b860144..ec76a8b1acc1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -32,7 +32,6 @@ #include #include -#include #include #include #include diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 41807ef55201..351ea0a41a03 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -3,7 +3,6 @@ #include #include -#include #include #include "mlx5_core.h" #include "mlx5_irq.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pd.c b/drivers/net/ethernet/mellanox/mlx5/core/pd.c index aabc53ad8bdd..ee5ffdeb9015 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pd.c @@ -31,7 +31,6 @@ */ #include -#include #include #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/rl.c b/drivers/net/ethernet/mellanox/mlx5/core/rl.c index 7161220afe30..9f8b4005f4bd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/rl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/rl.c @@ -31,7 +31,6 @@ */ #include -#include #include #include "mlx5_core.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c index 2e8b109fb34e..d232f1ea34a2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_dbg.c @@ -3,7 +3,6 @@ #include #include -#include #include #include "dr_types.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c index 01e9c412977c..9c81fb7c2c3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c @@ -31,7 +31,6 @@ */ #include -#include #include #include #include "mlx5_core.h" From 196df17ac53a715e55d99fda6025c2db22504bb2 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:59:55 +0200 Subject: [PATCH 03/15] net/mlx5: Node-aware allocation for the IRQ table Prefer the aware allocation, use the device NUMA node. Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 351ea0a41a03..db77f1d2eeb4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -600,7 +600,8 @@ int mlx5_irq_table_init(struct mlx5_core_dev *dev) if (mlx5_core_is_sf(dev)) return 0; - irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); + irq_table = kvzalloc_node(sizeof(*irq_table), GFP_KERNEL, + dev->priv.numa_node); if (!irq_table) return -ENOMEM; From e894246df5136f670f3568b7856daeec6c4f9456 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:53:14 +0200 Subject: [PATCH 04/15] net/mlx5: Node-aware allocation for the EQ table Prefer the aware allocation, use the device NUMA node. Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/eq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index cb4730cb456a..316105378188 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -438,7 +438,8 @@ int mlx5_eq_table_init(struct mlx5_core_dev *dev) struct mlx5_eq_table *eq_table; int i; - eq_table = kvzalloc(sizeof(*eq_table), GFP_KERNEL); + eq_table = kvzalloc_node(sizeof(*eq_table), GFP_KERNEL, + dev->priv.numa_node); if (!eq_table) return -ENOMEM; From 7f880719b953eba8c13b4e2ebe8c1f9d5fd44848 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:55:30 +0200 Subject: [PATCH 05/15] net/mlx5: Node-aware allocation for the EQs Prefer the aware allocation, use the device NUMA node. Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/eq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 316105378188..229728c80233 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -728,7 +728,8 @@ struct mlx5_eq * mlx5_eq_create_generic(struct mlx5_core_dev *dev, struct mlx5_eq_param *param) { - struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL); + struct mlx5_eq *eq = kvzalloc_node(sizeof(*eq), GFP_KERNEL, + dev->priv.numa_node); int err; if (!eq) @@ -888,10 +889,11 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) return ncomp_eqs; INIT_LIST_HEAD(&table->comp_eqs_list); nent = comp_eq_depth_devlink_param_get(dev); + for (i = 0; i < ncomp_eqs; i++) { struct mlx5_eq_param param = {}; - eq = kzalloc(sizeof(*eq), GFP_KERNEL); + eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, dev->priv.numa_node); if (!eq) { err = -ENOMEM; goto clean; From b5e4c3079490638dafa59413a51f9aef3df120d9 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:42:46 +0200 Subject: [PATCH 06/15] net/mlx5: Node-aware allocation for UAR Prefer the aware allocation, use the device NUMA node. Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/uar.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/uar.c b/drivers/net/ethernet/mellanox/mlx5/core/uar.c index 9c81fb7c2c3c..8455e79bc44a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/uar.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/uar.c @@ -99,19 +99,21 @@ static struct mlx5_uars_page *alloc_uars_page(struct mlx5_core_dev *mdev, int err = -ENOMEM; phys_addr_t pfn; int bfregs; + int node; int i; bfregs = uars_per_sys_page(mdev) * MLX5_BFREGS_PER_UAR; - up = kzalloc(sizeof(*up), GFP_KERNEL); + node = mdev->priv.numa_node; + up = kzalloc_node(sizeof(*up), GFP_KERNEL, node); if (!up) return ERR_PTR(err); up->mdev = mdev; - up->reg_bitmap = bitmap_zalloc(bfregs, GFP_KERNEL); + up->reg_bitmap = bitmap_zalloc_node(bfregs, GFP_KERNEL, node); if (!up->reg_bitmap) goto error1; - up->fp_bitmap = bitmap_zalloc(bfregs, GFP_KERNEL); + up->fp_bitmap = bitmap_zalloc_node(bfregs, GFP_KERNEL, node); if (!up->fp_bitmap) goto error1; From a3540effb766b868af429ae6f4909457b4d4f5ea Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 30 Dec 2020 11:45:34 +0200 Subject: [PATCH 07/15] net/mlx5: Node-aware allocation for the doorbell pgdir The function is node-aware and gets the node as an argument. Use a node-aware allocation for the doorbell pgdir structure. Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/alloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c index 291e427e9e4f..d5408f6ce5a7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c @@ -183,11 +183,11 @@ static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, u32 db_per_page = PAGE_SIZE / cache_line_size(); struct mlx5_db_pgdir *pgdir; - pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); + pgdir = kzalloc_node(sizeof(*pgdir), GFP_KERNEL, node); if (!pgdir) return NULL; - pgdir->bitmap = bitmap_zalloc(db_per_page, GFP_KERNEL); + pgdir->bitmap = bitmap_zalloc_node(db_per_page, GFP_KERNEL, node); if (!pgdir->bitmap) { kfree(pgdir); return NULL; From 7690900052303e758b797db1ff8013fe425d1806 Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Tue, 23 Nov 2021 14:41:18 +0200 Subject: [PATCH 08/15] net/mlx5: CT: Introduce a platform for multiple flow steering providers Currently, fs_core layer provides flow steering services to the driver including: autogroups, allocating FTEs (flow table entries) and FTE ids, and support of fte action modification. If then software steering is configured, rule insertion will go through a translation layer from firmware buffers to software steering objects (see fs_dr.c). The connection tracking table is a system table that is not directly controlled by the user and is a very high scale table. These fs_core services introduces an overhead that may be optimized by using software steering API directly. Introduce ct flow steering interface to allow multiple flow steering providers. Use the new interface to implement the current dmfs (device managed flow steering) provider which uses fs_core insertion. Signed-off-by: Paul Blakey Reviewed-by: Oz Shlomo Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../ethernet/mellanox/mlx5/core/en/tc/ct_fs.h | 24 +++++++ .../mellanox/mlx5/core/en/tc/ct_fs_dmfs.c | 64 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/en/tc_ct.c | 33 +++++++++- 4 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index b94cca45eb78..24cf559811b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -55,7 +55,7 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/a en/tc/act/ct.o en/tc/act/sample.o en/tc/act/ptype.o \ en/tc/act/redirect_ingress.o -mlx5_core-$(CONFIG_MLX5_TC_CT) += en/tc_ct.o +mlx5_core-$(CONFIG_MLX5_TC_CT) += en/tc_ct.o en/tc/ct_fs_dmfs.o mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += en/tc/sample.o # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h new file mode 100644 index 000000000000..0df12d490f2b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#ifndef __MLX5_EN_TC_CT_FS_H__ +#define __MLX5_EN_TC_CT_FS_H__ + +struct mlx5_ct_fs { + const struct net_device *netdev; + struct mlx5_core_dev *dev; +}; + +struct mlx5_ct_fs_rule { +}; + +struct mlx5_ct_fs_ops { + struct mlx5_ct_fs_rule * (*ct_rule_add)(struct mlx5_ct_fs *fs, + struct mlx5_flow_spec *spec, + struct mlx5_flow_attr *attr); + void (*ct_rule_del)(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule); +}; + +struct mlx5_ct_fs_ops *mlx5_ct_fs_dmfs_ops_get(void); + +#endif /* __MLX5_EN_TC_CT_FS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c new file mode 100644 index 000000000000..94bd525ca62e --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#include "en_tc.h" +#include "en/tc_ct.h" +#include "en/tc/ct_fs.h" + +#define ct_dbg(fmt, args...)\ + netdev_dbg(fs->netdev, "ct_fs_dmfs debug: " fmt "\n", ##args) + +struct mlx5_ct_fs_dmfs_rule { + struct mlx5_ct_fs_rule fs_rule; + struct mlx5_flow_handle *rule; + struct mlx5_flow_attr *attr; +}; + +static struct mlx5_ct_fs_rule * +mlx5_ct_fs_dmfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, + struct mlx5_flow_attr *attr) +{ + struct mlx5e_priv *priv = netdev_priv(fs->netdev); + struct mlx5_ct_fs_dmfs_rule *dmfs_rule; + int err; + + dmfs_rule = kzalloc(sizeof(*dmfs_rule), GFP_KERNEL); + if (!dmfs_rule) + return ERR_PTR(-ENOMEM); + + dmfs_rule->rule = mlx5_tc_rule_insert(priv, spec, attr); + if (IS_ERR(dmfs_rule->rule)) { + err = PTR_ERR(dmfs_rule->rule); + ct_dbg("Failed to add ct entry fs rule"); + goto err_insert; + } + + dmfs_rule->attr = attr; + + return &dmfs_rule->fs_rule; + +err_insert: + kfree(dmfs_rule); + return ERR_PTR(err); +} + +static void +mlx5_ct_fs_dmfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule) +{ + struct mlx5_ct_fs_dmfs_rule *dmfs_rule = container_of(fs_rule, + struct mlx5_ct_fs_dmfs_rule, + fs_rule); + + mlx5_tc_rule_delete(netdev_priv(fs->netdev), dmfs_rule->rule, dmfs_rule->attr); + kfree(dmfs_rule); +} + +static struct mlx5_ct_fs_ops dmfs_ops = { + .ct_rule_add = mlx5_ct_fs_dmfs_ct_rule_add, + .ct_rule_del = mlx5_ct_fs_dmfs_ct_rule_del, +}; + +struct mlx5_ct_fs_ops *mlx5_ct_fs_dmfs_ops_get(void) +{ + return &dmfs_ops; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 875e77af0ae6..09e88d8e17d3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -18,6 +18,7 @@ #include "lib/fs_chains.h" #include "en/tc_ct.h" +#include "en/tc/ct_fs.h" #include "en/tc_priv.h" #include "en/mod_hdr.h" #include "en/mapping.h" @@ -63,6 +64,8 @@ struct mlx5_tc_ct_priv { struct mapping_ctx *labels_mapping; enum mlx5_flow_namespace_type ns_type; struct mlx5_fs_chains *chains; + struct mlx5_ct_fs *fs; + struct mlx5_ct_fs_ops *fs_ops; spinlock_t ht_lock; /* protects ft entries */ }; @@ -74,7 +77,7 @@ struct mlx5_ct_flow { }; struct mlx5_ct_zone_rule { - struct mlx5_flow_handle *rule; + struct mlx5_ct_fs_rule *rule; struct mlx5e_mod_hdr_handle *mh; struct mlx5_flow_attr *attr; bool nat; @@ -505,7 +508,7 @@ mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv, ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone); - mlx5_tc_rule_delete(netdev_priv(ct_priv->netdev), zone_rule->rule, attr); + ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule); mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh); mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); kfree(attr); @@ -816,7 +819,7 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule); mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK); - zone_rule->rule = mlx5_tc_rule_insert(priv, spec, attr); + zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr); if (IS_ERR(zone_rule->rule)) { err = PTR_ERR(zone_rule->rule); ct_dbg("Failed to add ct entry rule, nat: %d", nat); @@ -1960,6 +1963,22 @@ mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv, mutex_unlock(&priv->control_lock); } +static int +mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv) +{ + struct mlx5_ct_fs_ops *fs_ops = mlx5_ct_fs_dmfs_ops_get(); + + ct_priv->fs = kzalloc(sizeof(*ct_priv->fs), GFP_KERNEL); + if (!ct_priv->fs) + return -ENOMEM; + + ct_priv->fs->netdev = ct_priv->netdev; + ct_priv->fs->dev = ct_priv->dev; + ct_priv->fs_ops = fs_ops; + + return 0; +} + static int mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw, const char **err_msg) @@ -2098,8 +2117,14 @@ mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, if (rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params)) goto err_ct_tuples_nat_ht; + err = mlx5_tc_ct_fs_init(ct_priv); + if (err) + goto err_init_fs; + return ct_priv; +err_init_fs: + rhashtable_destroy(&ct_priv->ct_tuples_nat_ht); err_ct_tuples_nat_ht: rhashtable_destroy(&ct_priv->ct_tuples_ht); err_ct_tuples_ht: @@ -2130,6 +2155,8 @@ mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv) chains = ct_priv->chains; + kfree(ct_priv->fs); + mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat); mlx5_chains_destroy_global_table(chains, ct_priv->ct); mapping_destroy(ct_priv->zone_mapping); From 34ea969d164503b683d6b918b13b78a8e267c2cf Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Tue, 13 Jul 2021 11:19:24 +0300 Subject: [PATCH 09/15] net/mlx5: DR, Add helper to get backing dr table from a mlx5 flow table If sw steering was used to create the table, dr steeering fs creates a backing dr table for the mlx5 flow table. Add helper to return this table so it can be used to create matchers and add rules on it directly instead of passing via eswitch_offloads/fs_core insertion. Signed-off-by: Paul Blakey Reviewed-by: Oz Shlomo Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c | 5 +++++ drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h | 3 +++ 2 files changed, 8 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c index 8ca110643cc0..f5f2d356e75f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c @@ -305,3 +305,8 @@ u32 mlx5dr_table_get_id(struct mlx5dr_table *tbl) { return tbl->table_id; } + +struct mlx5dr_table *mlx5dr_table_get_from_fs_ft(struct mlx5_flow_table *ft) +{ + return ft->fs_dr_table.dr_table; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 03efbdf3fec3..ec5cbec0d455 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -53,6 +53,9 @@ void mlx5dr_domain_set_peer(struct mlx5dr_domain *dmn, struct mlx5dr_table * mlx5dr_table_create(struct mlx5dr_domain *domain, u32 level, u32 flags); +struct mlx5dr_table * +mlx5dr_table_get_from_fs_ft(struct mlx5_flow_table *ft); + int mlx5dr_table_destroy(struct mlx5dr_table *table); u32 mlx5dr_table_get_id(struct mlx5dr_table *table); From c6fef514adaa0d681f486ebbdec4c299db7101d6 Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Thu, 30 Sep 2021 12:06:35 +0300 Subject: [PATCH 10/15] net/mlx5: Add smfs lib to export direct steering API to CT Add a thin layer that exports selected direct steering (dr) API which will be used by a ct fs implementation in a following patch. Signed-off-by: Paul Blakey Reviewed-by: Mark Bloch Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../ethernet/mellanox/mlx5/core/lib/smfs.c | 68 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/lib/smfs.h | 36 ++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.c create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 24cf559811b0..f68a9db18698 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -106,7 +106,7 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o steering/dr_ste_v2.o \ steering/dr_cmd.o steering/dr_fw.o \ steering/dr_action.o steering/fs_dr.o \ - steering/dr_dbg.o + steering/dr_dbg.o lib/smfs.o # # SF device # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.c new file mode 100644 index 000000000000..9b8c051ccf65 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#include +#include + +#include "smfs.h" + +struct mlx5dr_matcher * +mlx5_smfs_matcher_create(struct mlx5dr_table *table, u32 priority, struct mlx5_flow_spec *spec) +{ + struct mlx5dr_match_parameters matcher_mask = {}; + + matcher_mask.match_buf = (u64 *)&spec->match_criteria; + matcher_mask.match_sz = DR_SZ_MATCH_PARAM; + + return mlx5dr_matcher_create(table, priority, spec->match_criteria_enable, &matcher_mask); +} + +void +mlx5_smfs_matcher_destroy(struct mlx5dr_matcher *matcher) +{ + mlx5dr_matcher_destroy(matcher); +} + +struct mlx5dr_table * +mlx5_smfs_table_get_from_fs_ft(struct mlx5_flow_table *ft) +{ + return mlx5dr_table_get_from_fs_ft(ft); +} + +struct mlx5dr_action * +mlx5_smfs_action_create_dest_table(struct mlx5dr_table *table) +{ + return mlx5dr_action_create_dest_table(table); +} + +struct mlx5dr_action * +mlx5_smfs_action_create_flow_counter(u32 counter_id) +{ + return mlx5dr_action_create_flow_counter(counter_id); +} + +void +mlx5_smfs_action_destroy(struct mlx5dr_action *action) +{ + mlx5dr_action_destroy(action); +} + +struct mlx5dr_rule * +mlx5_smfs_rule_create(struct mlx5dr_matcher *matcher, struct mlx5_flow_spec *spec, + size_t num_actions, struct mlx5dr_action *actions[], + u32 flow_source) +{ + struct mlx5dr_match_parameters value = {}; + + value.match_buf = (u64 *)spec->match_value; + value.match_sz = DR_SZ_MATCH_PARAM; + + return mlx5dr_rule_create(matcher, &value, num_actions, actions, flow_source); +} + +void +mlx5_smfs_rule_destroy(struct mlx5dr_rule *rule) +{ + mlx5dr_rule_destroy(rule); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.h new file mode 100644 index 000000000000..452d0df339ac --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/smfs.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#ifndef __MLX5_LIB_SMFS_H__ +#define __MLX5_LIB_SMFS_H__ + +#include "steering/mlx5dr.h" +#include "steering/dr_types.h" + +struct mlx5dr_matcher * +mlx5_smfs_matcher_create(struct mlx5dr_table *table, u32 priority, struct mlx5_flow_spec *spec); + +void +mlx5_smfs_matcher_destroy(struct mlx5dr_matcher *matcher); + +struct mlx5dr_table * +mlx5_smfs_table_get_from_fs_ft(struct mlx5_flow_table *ft); + +struct mlx5dr_action * +mlx5_smfs_action_create_dest_table(struct mlx5dr_table *table); + +struct mlx5dr_action * +mlx5_smfs_action_create_flow_counter(u32 counter_id); + +void +mlx5_smfs_action_destroy(struct mlx5dr_action *action); + +struct mlx5dr_rule * +mlx5_smfs_rule_create(struct mlx5dr_matcher *matcher, struct mlx5_flow_spec *spec, + size_t num_actions, struct mlx5dr_action *actions[], + u32 flow_source); + +void +mlx5_smfs_rule_destroy(struct mlx5dr_rule *rule); + +#endif /* __MLX5_LIB_SMFS_H__ */ From 3ee61ebb0df1ec5cf9b60a12c1d7249230a73f9c Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Wed, 29 Sep 2021 13:56:56 +0300 Subject: [PATCH 11/15] net/mlx5: CT: Add software steering ct flow steering provider fs_core layer adds extra book keeping that is either unneeded for CT, or unused by the underlying software steering, such as allocating FTEs and FTE ids, saving the match key and mask, and autogroups management. On top of that, direct steering has a translation layer (fs_dr) from PRM commands to direct steering objects, for example, creating temporary dr_action objects. This has a performance impact when dealing with CT high insertion rate. To use direct steering (smfs) directly for ct, add a tc ct fs smfs implementation. Instead of dmfs autogroups, smfs ct fs uses one of 4 predefined dr matchers in CT and CT-NAT tables, for each combination of tuple ethertype (ipv4/ipv6), and tuple ip_proto (udp/tcp) that is currently used by nf flow table flow offload. At rule insertions, validate the flow rule fits one of the predfined matcher, and insert to it. To fill the dr_actions of the rule efficiently, create the fwd to post_ct tbl dr_action at fs init, the count dr_action at counter creation, and re-use the already pre-allocated modify header dr_action. Signed-off-by: Paul Blakey Reviewed-by: Oz Shlomo Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/Makefile | 6 +- .../ethernet/mellanox/mlx5/core/en/tc/ct_fs.h | 27 +- .../mellanox/mlx5/core/en/tc/ct_fs_dmfs.c | 17 +- .../mellanox/mlx5/core/en/tc/ct_fs_smfs.c | 366 ++++++++++++++++++ .../ethernet/mellanox/mlx5/core/en/tc_ct.c | 24 +- .../ethernet/mellanox/mlx5/core/en/tc_ct.h | 2 + 6 files changed, 435 insertions(+), 7 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index f68a9db18698..4bc666714a35 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -55,7 +55,11 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/a en/tc/act/ct.o en/tc/act/sample.o en/tc/act/ptype.o \ en/tc/act/redirect_ingress.o -mlx5_core-$(CONFIG_MLX5_TC_CT) += en/tc_ct.o en/tc/ct_fs_dmfs.o +ifneq ($(CONFIG_MLX5_TC_CT),) + mlx5_core-y += en/tc_ct.o en/tc/ct_fs_dmfs.o + mlx5_core-$(CONFIG_MLX5_SW_STEERING) += en/tc/ct_fs_smfs.o +endif + mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += en/tc/sample.o # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h index 0df12d490f2b..bb6b1a979ba1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs.h @@ -7,18 +7,43 @@ struct mlx5_ct_fs { const struct net_device *netdev; struct mlx5_core_dev *dev; + + /* private data */ + void *priv_data[]; }; struct mlx5_ct_fs_rule { }; struct mlx5_ct_fs_ops { + int (*init)(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct, + struct mlx5_flow_table *ct_nat, struct mlx5_flow_table *post_ct); + void (*destroy)(struct mlx5_ct_fs *fs); + struct mlx5_ct_fs_rule * (*ct_rule_add)(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, - struct mlx5_flow_attr *attr); + struct mlx5_flow_attr *attr, + struct flow_rule *flow_rule); void (*ct_rule_del)(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule); + + size_t priv_size; }; +static inline void *mlx5_ct_fs_priv(struct mlx5_ct_fs *fs) +{ + return &fs->priv_data; +} + struct mlx5_ct_fs_ops *mlx5_ct_fs_dmfs_ops_get(void); +#if IS_ENABLED(CONFIG_MLX5_SW_STEERING) +struct mlx5_ct_fs_ops *mlx5_ct_fs_smfs_ops_get(void); +#else +static inline struct mlx5_ct_fs_ops * +mlx5_ct_fs_smfs_ops_get(void) +{ + return NULL; +} +#endif /* IS_ENABLED(CONFIG_MLX5_SW_STEERING) */ + #endif /* __MLX5_EN_TC_CT_FS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c index 94bd525ca62e..ae4f55be48ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_dmfs.c @@ -14,9 +14,21 @@ struct mlx5_ct_fs_dmfs_rule { struct mlx5_flow_attr *attr; }; +static int +mlx5_ct_fs_dmfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct, + struct mlx5_flow_table *ct_nat, struct mlx5_flow_table *post_ct) +{ + return 0; +} + +static void +mlx5_ct_fs_dmfs_destroy(struct mlx5_ct_fs *fs) +{ +} + static struct mlx5_ct_fs_rule * mlx5_ct_fs_dmfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, - struct mlx5_flow_attr *attr) + struct mlx5_flow_attr *attr, struct flow_rule *flow_rule) { struct mlx5e_priv *priv = netdev_priv(fs->netdev); struct mlx5_ct_fs_dmfs_rule *dmfs_rule; @@ -56,6 +68,9 @@ mlx5_ct_fs_dmfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_ru static struct mlx5_ct_fs_ops dmfs_ops = { .ct_rule_add = mlx5_ct_fs_dmfs_ct_rule_add, .ct_rule_del = mlx5_ct_fs_dmfs_ct_rule_del, + + .init = mlx5_ct_fs_dmfs_init, + .destroy = mlx5_ct_fs_dmfs_destroy, }; struct mlx5_ct_fs_ops *mlx5_ct_fs_dmfs_ops_get(void) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c new file mode 100644 index 000000000000..6da8b87c0475 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#include "en_tc.h" +#include "en/tc_priv.h" +#include "en/tc_ct.h" +#include "en/tc/ct_fs.h" + +#include "lib/smfs.h" + +#define INIT_ERR_PREFIX "ct_fs_smfs init failed" +#define ct_dbg(fmt, args...)\ + netdev_dbg(fs->netdev, "ct_fs_smfs debug: " fmt "\n", ##args) +#define MLX5_CT_TCP_FLAGS_MASK cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16) + +struct mlx5_ct_fs_smfs_matchers { + struct mlx5dr_matcher *ipv4_tcp; + struct mlx5dr_matcher *ipv4_udp; + struct mlx5dr_matcher *ipv6_tcp; + struct mlx5dr_matcher *ipv6_udp; +}; + +struct mlx5_ct_fs_smfs { + struct mlx5_ct_fs_smfs_matchers ct_matchers; + struct mlx5_ct_fs_smfs_matchers ct_matchers_nat; + struct mlx5dr_action *fwd_action; + struct mlx5_flow_table *ct_nat; +}; + +struct mlx5_ct_fs_smfs_rule { + struct mlx5_ct_fs_rule fs_rule; + struct mlx5dr_rule *rule; + struct mlx5dr_action *count_action; +}; + +static inline void +mlx5_ct_fs_smfs_fill_mask(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, bool ipv4, bool tcp) +{ + void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers); + + if (likely(MLX5_CAP_FLOWTABLE_NIC_RX(fs->dev, ft_field_support.outer_ip_version))) + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_version); + else + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ethertype); + + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol); + if (likely(ipv4)) { + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, + src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, + dst_ipv4_dst_ipv6.ipv4_layout.ipv4); + } else { + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6), + 0xFF, + MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4, + dst_ipv4_dst_ipv6.ipv6_layout.ipv6)); + memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, + src_ipv4_src_ipv6.ipv6_layout.ipv6), + 0xFF, + MLX5_FLD_SZ_BYTES(fte_match_set_lyr_2_4, + src_ipv4_src_ipv6.ipv6_layout.ipv6)); + } + + if (likely(tcp)) { + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_sport); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, tcp_dport); + MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags, + ntohs(MLX5_CT_TCP_FLAGS_MASK)); + } else { + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_sport); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, udp_dport); + } + + mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, 0, MLX5_CT_ZONE_MASK); +} + +static struct mlx5dr_matcher * +mlx5_ct_fs_smfs_matcher_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, bool ipv4, + bool tcp, u32 priority) +{ + struct mlx5dr_matcher *dr_matcher; + struct mlx5_flow_spec *spec; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return ERR_PTR(-ENOMEM); + + mlx5_ct_fs_smfs_fill_mask(fs, spec, ipv4, tcp); + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2 | MLX5_MATCH_OUTER_HEADERS; + + dr_matcher = mlx5_smfs_matcher_create(tbl, priority, spec); + kfree(spec); + if (!dr_matcher) + return ERR_PTR(-EINVAL); + + return dr_matcher; +} + +static int +mlx5_ct_fs_smfs_matchers_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, + struct mlx5_ct_fs_smfs_matchers *ct_matchers) +{ + const struct net_device *netdev = fs->netdev; + u32 prio = 0; + int err; + + ct_matchers->ipv4_tcp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, true, true, prio); + if (IS_ERR(ct_matchers->ipv4_tcp)) { + err = PTR_ERR(ct_matchers->ipv4_tcp); + netdev_warn(netdev, + "%s, failed to create ipv4 tcp matcher, err: %d\n", + INIT_ERR_PREFIX, err); + return err; + } + + ++prio; + ct_matchers->ipv4_udp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, true, false, prio); + if (IS_ERR(ct_matchers->ipv4_udp)) { + err = PTR_ERR(ct_matchers->ipv4_udp); + netdev_warn(netdev, + "%s, failed to create ipv4 udp matcher, err: %d\n", + INIT_ERR_PREFIX, err); + goto err_matcher_ipv4_udp; + } + + ++prio; + ct_matchers->ipv6_tcp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, false, true, prio); + if (IS_ERR(ct_matchers->ipv6_tcp)) { + err = PTR_ERR(ct_matchers->ipv6_tcp); + netdev_warn(netdev, + "%s, failed to create ipv6 tcp matcher, err: %d\n", + INIT_ERR_PREFIX, err); + goto err_matcher_ipv6_tcp; + } + + ++prio; + ct_matchers->ipv6_udp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, false, false, prio); + if (IS_ERR(ct_matchers->ipv6_udp)) { + err = PTR_ERR(ct_matchers->ipv6_udp); + netdev_warn(netdev, + "%s, failed to create ipv6 tcp matcher, err: %d\n", + INIT_ERR_PREFIX, err); + goto err_matcher_ipv6_udp; + } + + return 0; + +err_matcher_ipv6_udp: + mlx5_smfs_matcher_destroy(ct_matchers->ipv6_tcp); +err_matcher_ipv6_tcp: + mlx5_smfs_matcher_destroy(ct_matchers->ipv4_udp); +err_matcher_ipv4_udp: + mlx5_smfs_matcher_destroy(ct_matchers->ipv4_tcp); + return 0; +} + +static void +mlx5_ct_fs_smfs_matchers_destroy(struct mlx5_ct_fs_smfs_matchers *ct_matchers) +{ + mlx5_smfs_matcher_destroy(ct_matchers->ipv6_udp); + mlx5_smfs_matcher_destroy(ct_matchers->ipv6_tcp); + mlx5_smfs_matcher_destroy(ct_matchers->ipv4_udp); + mlx5_smfs_matcher_destroy(ct_matchers->ipv4_tcp); +} + +static int +mlx5_ct_fs_smfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct, + struct mlx5_flow_table *ct_nat, struct mlx5_flow_table *post_ct) +{ + struct mlx5dr_table *ct_tbl, *ct_nat_tbl, *post_ct_tbl; + struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); + int err; + + post_ct_tbl = mlx5_smfs_table_get_from_fs_ft(post_ct); + ct_nat_tbl = mlx5_smfs_table_get_from_fs_ft(ct_nat); + ct_tbl = mlx5_smfs_table_get_from_fs_ft(ct); + fs_smfs->ct_nat = ct_nat; + + if (!ct_tbl || !ct_nat_tbl || !post_ct_tbl) { + netdev_warn(fs->netdev, "ct_fs_smfs: failed to init, missing backing dr tables"); + return -EOPNOTSUPP; + } + + ct_dbg("using smfs steering"); + + err = mlx5_ct_fs_smfs_matchers_create(fs, ct_tbl, &fs_smfs->ct_matchers); + if (err) + goto err_init; + + err = mlx5_ct_fs_smfs_matchers_create(fs, ct_nat_tbl, &fs_smfs->ct_matchers_nat); + if (err) + goto err_matchers_nat; + + fs_smfs->fwd_action = mlx5_smfs_action_create_dest_table(post_ct_tbl); + if (!fs_smfs->fwd_action) { + err = -EINVAL; + goto err_action_create; + } + + return 0; + +err_action_create: + mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers_nat); +err_matchers_nat: + mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers); +err_init: + return err; +} + +static void +mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs *fs) +{ + struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); + + mlx5_smfs_action_destroy(fs_smfs->fwd_action); + mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers_nat); + mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers); +} + +static inline bool +mlx5_tc_ct_valid_used_dissector_keys(const u32 used_keys) +{ +#define DISSECTOR_BIT(name) BIT(FLOW_DISSECTOR_KEY_ ## name) + const u32 basic_keys = DISSECTOR_BIT(BASIC) | DISSECTOR_BIT(CONTROL) | + DISSECTOR_BIT(PORTS) | DISSECTOR_BIT(META); + const u32 ipv4_tcp = basic_keys | DISSECTOR_BIT(IPV4_ADDRS) | DISSECTOR_BIT(TCP); + const u32 ipv4_udp = basic_keys | DISSECTOR_BIT(IPV4_ADDRS); + const u32 ipv6_tcp = basic_keys | DISSECTOR_BIT(IPV6_ADDRS) | DISSECTOR_BIT(TCP); + const u32 ipv6_udp = basic_keys | DISSECTOR_BIT(IPV6_ADDRS); + + return (used_keys == ipv4_tcp || used_keys == ipv4_udp || used_keys == ipv6_tcp || + used_keys == ipv6_udp); +} + +static bool +mlx5_ct_fs_smfs_ct_validate_flow_rule(struct mlx5_ct_fs *fs, struct flow_rule *flow_rule) +{ + struct flow_match_ipv4_addrs ipv4_addrs; + struct flow_match_ipv6_addrs ipv6_addrs; + struct flow_match_control control; + struct flow_match_basic basic; + struct flow_match_ports ports; + struct flow_match_tcp tcp; + + if (!mlx5_tc_ct_valid_used_dissector_keys(flow_rule->match.dissector->used_keys)) { + ct_dbg("rule uses unexpected dissectors (0x%08x)", + flow_rule->match.dissector->used_keys); + return false; + } + + flow_rule_match_basic(flow_rule, &basic); + flow_rule_match_control(flow_rule, &control); + flow_rule_match_ipv4_addrs(flow_rule, &ipv4_addrs); + flow_rule_match_ipv6_addrs(flow_rule, &ipv6_addrs); + flow_rule_match_ports(flow_rule, &ports); + flow_rule_match_tcp(flow_rule, &tcp); + + if (basic.mask->n_proto != htons(0xFFFF) || + (basic.key->n_proto != htons(ETH_P_IP) && basic.key->n_proto != htons(ETH_P_IPV6)) || + basic.mask->ip_proto != 0xFF || + (basic.key->ip_proto != IPPROTO_UDP && basic.key->ip_proto != IPPROTO_TCP)) { + ct_dbg("rule uses unexpected basic match (n_proto 0x%04x/0x%04x, ip_proto 0x%02x/0x%02x)", + ntohs(basic.key->n_proto), ntohs(basic.mask->n_proto), + basic.key->ip_proto, basic.mask->ip_proto); + return false; + } + + if (ports.mask->src != htons(0xFFFF) || ports.mask->dst != htons(0xFFFF)) { + ct_dbg("rule uses ports match (src 0x%04x, dst 0x%04x)", + ports.mask->src, ports.mask->dst); + return false; + } + + if (basic.key->ip_proto == IPPROTO_TCP && tcp.mask->flags != MLX5_CT_TCP_FLAGS_MASK) { + ct_dbg("rule uses unexpected tcp match (flags 0x%02x)", tcp.mask->flags); + return false; + } + + return true; +} + +static struct mlx5_ct_fs_rule * +mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, + struct mlx5_flow_attr *attr, struct flow_rule *flow_rule) +{ + struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); + struct mlx5_ct_fs_smfs_matchers *matchers; + struct mlx5_ct_fs_smfs_rule *smfs_rule; + struct mlx5dr_action *actions[5]; + struct mlx5dr_matcher *matcher; + struct mlx5dr_rule *rule; + int num_actions = 0, err; + bool nat, tcp, ipv4; + + if (!mlx5_ct_fs_smfs_ct_validate_flow_rule(fs, flow_rule)) + return ERR_PTR(-EOPNOTSUPP); + + smfs_rule = kzalloc(sizeof(*smfs_rule), GFP_KERNEL); + if (!smfs_rule) + return ERR_PTR(-ENOMEM); + + smfs_rule->count_action = mlx5_smfs_action_create_flow_counter(mlx5_fc_id(attr->counter)); + if (!smfs_rule->count_action) { + err = -EINVAL; + goto err_count; + } + + actions[num_actions++] = smfs_rule->count_action; + actions[num_actions++] = attr->modify_hdr->action.dr_action; + actions[num_actions++] = fs_smfs->fwd_action; + + nat = (attr->ft == fs_smfs->ct_nat); + ipv4 = mlx5e_tc_get_ip_version(spec, true) == 4; + tcp = MLX5_GET(fte_match_param, spec->match_value, + outer_headers.ip_protocol) == IPPROTO_TCP; + + matchers = nat ? &fs_smfs->ct_matchers_nat : &fs_smfs->ct_matchers; + matcher = ipv4 ? (tcp ? matchers->ipv4_tcp : matchers->ipv4_udp) : + (tcp ? matchers->ipv6_tcp : matchers->ipv6_udp); + + rule = mlx5_smfs_rule_create(matcher, spec, num_actions, actions, + MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT); + if (!rule) { + err = -EINVAL; + goto err_rule; + } + + smfs_rule->rule = rule; + + return &smfs_rule->fs_rule; + +err_rule: + mlx5_smfs_action_destroy(smfs_rule->count_action); +err_count: + kfree(smfs_rule); + return ERR_PTR(err); +} + +static void +mlx5_ct_fs_smfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_rule) +{ + struct mlx5_ct_fs_smfs_rule *smfs_rule = container_of(fs_rule, + struct mlx5_ct_fs_smfs_rule, + fs_rule); + + mlx5_smfs_rule_destroy(smfs_rule->rule); + mlx5_smfs_action_destroy(smfs_rule->count_action); + kfree(smfs_rule); +} + +static struct mlx5_ct_fs_ops fs_smfs_ops = { + .ct_rule_add = mlx5_ct_fs_smfs_ct_rule_add, + .ct_rule_del = mlx5_ct_fs_smfs_ct_rule_del, + + .init = mlx5_ct_fs_smfs_init, + .destroy = mlx5_ct_fs_smfs_destroy, + + .priv_size = sizeof(struct mlx5_ct_fs_smfs), +}; + +struct mlx5_ct_fs_ops * +mlx5_ct_fs_smfs_ops_get(void) +{ + return &fs_smfs_ops; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 09e88d8e17d3..ca1510399d1e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -26,9 +26,8 @@ #include "en.h" #include "en_tc.h" #include "en_rep.h" +#include "fs_core.h" -#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen) -#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0) #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1) #define MLX5_CT_STATE_TRK_BIT BIT(2) #define MLX5_CT_STATE_NAT_BIT BIT(3) @@ -819,7 +818,7 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule); mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK); - zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr); + zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule); if (IS_ERR(zone_rule->rule)) { err = PTR_ERR(zone_rule->rule); ct_dbg("Failed to add ct entry rule, nat: %d", nat); @@ -1966,9 +1965,17 @@ mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv, static int mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv) { + struct mlx5_flow_table *post_ct = mlx5e_tc_post_act_get_ft(ct_priv->post_act); struct mlx5_ct_fs_ops *fs_ops = mlx5_ct_fs_dmfs_ops_get(); + int err; - ct_priv->fs = kzalloc(sizeof(*ct_priv->fs), GFP_KERNEL); + if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB && + ct_priv->dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS) { + ct_dbg("Using SMFS ct flow steering provider"); + fs_ops = mlx5_ct_fs_smfs_ops_get(); + } + + ct_priv->fs = kzalloc(sizeof(*ct_priv->fs) + fs_ops->priv_size, GFP_KERNEL); if (!ct_priv->fs) return -ENOMEM; @@ -1976,7 +1983,15 @@ mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv) ct_priv->fs->dev = ct_priv->dev; ct_priv->fs_ops = fs_ops; + err = ct_priv->fs_ops->init(ct_priv->fs, ct_priv->ct, ct_priv->ct_nat, post_ct); + if (err) + goto err_init; + return 0; + +err_init: + kfree(ct_priv->fs); + return err; } static int @@ -2155,6 +2170,7 @@ mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv) chains = ct_priv->chains; + ct_priv->fs_ops->destroy(ct_priv->fs); kfree(ct_priv->fs); mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 2b21c7b97a52..36d3652bf829 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -86,6 +86,8 @@ struct mlx5_ct_attr { #define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) #define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset) +#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen) +#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0) #if IS_ENABLED(CONFIG_MLX5_TC_CT) From fbf6836db42d013056e0a423599d8afeb5b838ac Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Sun, 5 Sep 2021 10:47:56 +0300 Subject: [PATCH 12/15] net/mlx5: CT: Create smfs dr matchers dynamically SMFS dr matchers are processed sequentially in hardware according to their priorities, and not skipped if empty. Currently, smfs ct fs creates four predefined dr matchers per ct table (ct/ct nat) with hardcoded priority. Compared to dmfs ct fs using autogroups, this might cause additional hops in fastpath for traffic patterns that match later priorties, even if previous priorites are empty, e.g user only using ipv6 UDP traffic will have additional 3 hops. Create the matchers dynamically, using the highest priority available, on first rule usage, and remove them on last usage. Signed-off-by: Paul Blakey Reviewed-by: Oz Shlomo Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en/tc/ct_fs_smfs.c | 182 +++++++++--------- 1 file changed, 94 insertions(+), 88 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c index 6da8b87c0475..59988e24b704 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/ct_fs_smfs.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ +#include + #include "en_tc.h" #include "en/tc_priv.h" #include "en/tc_ct.h" @@ -13,24 +15,32 @@ netdev_dbg(fs->netdev, "ct_fs_smfs debug: " fmt "\n", ##args) #define MLX5_CT_TCP_FLAGS_MASK cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16) +struct mlx5_ct_fs_smfs_matcher { + struct mlx5dr_matcher *dr_matcher; + struct list_head list; + int prio; + refcount_t ref; +}; + struct mlx5_ct_fs_smfs_matchers { - struct mlx5dr_matcher *ipv4_tcp; - struct mlx5dr_matcher *ipv4_udp; - struct mlx5dr_matcher *ipv6_tcp; - struct mlx5dr_matcher *ipv6_udp; + struct mlx5_ct_fs_smfs_matcher smfs_matchers[4]; + struct list_head used; }; struct mlx5_ct_fs_smfs { - struct mlx5_ct_fs_smfs_matchers ct_matchers; - struct mlx5_ct_fs_smfs_matchers ct_matchers_nat; + struct mlx5dr_table *ct_tbl, *ct_nat_tbl; + struct mlx5_ct_fs_smfs_matchers matchers; + struct mlx5_ct_fs_smfs_matchers matchers_nat; struct mlx5dr_action *fwd_action; struct mlx5_flow_table *ct_nat; + struct mutex lock; /* Guards matchers */ }; struct mlx5_ct_fs_smfs_rule { struct mlx5_ct_fs_rule fs_rule; struct mlx5dr_rule *rule; struct mlx5dr_action *count_action; + struct mlx5_ct_fs_smfs_matcher *smfs_matcher; }; static inline void @@ -97,71 +107,75 @@ mlx5_ct_fs_smfs_matcher_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, return dr_matcher; } -static int -mlx5_ct_fs_smfs_matchers_create(struct mlx5_ct_fs *fs, struct mlx5dr_table *tbl, - struct mlx5_ct_fs_smfs_matchers *ct_matchers) +static struct mlx5_ct_fs_smfs_matcher * +mlx5_ct_fs_smfs_matcher_get(struct mlx5_ct_fs *fs, bool nat, bool ipv4, bool tcp) { - const struct net_device *netdev = fs->netdev; - u32 prio = 0; - int err; + struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); + struct mlx5_ct_fs_smfs_matcher *m, *smfs_matcher; + struct mlx5_ct_fs_smfs_matchers *matchers; + struct mlx5dr_matcher *dr_matcher; + struct mlx5dr_table *tbl; + struct list_head *prev; + int prio; - ct_matchers->ipv4_tcp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, true, true, prio); - if (IS_ERR(ct_matchers->ipv4_tcp)) { - err = PTR_ERR(ct_matchers->ipv4_tcp); - netdev_warn(netdev, - "%s, failed to create ipv4 tcp matcher, err: %d\n", - INIT_ERR_PREFIX, err); - return err; + matchers = nat ? &fs_smfs->matchers_nat : &fs_smfs->matchers; + smfs_matcher = &matchers->smfs_matchers[ipv4 * 2 + tcp]; + + if (refcount_inc_not_zero(&smfs_matcher->ref)) + return smfs_matcher; + + mutex_lock(&fs_smfs->lock); + + /* Retry with lock, as another thread might have already created the relevant matcher + * till we acquired the lock + */ + if (refcount_inc_not_zero(&smfs_matcher->ref)) + goto out_unlock; + + // Find next available priority in sorted used list + prio = 0; + prev = &matchers->used; + list_for_each_entry(m, &matchers->used, list) { + prev = &m->list; + + if (m->prio == prio) + prio = m->prio + 1; + else + break; } - ++prio; - ct_matchers->ipv4_udp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, true, false, prio); - if (IS_ERR(ct_matchers->ipv4_udp)) { - err = PTR_ERR(ct_matchers->ipv4_udp); - netdev_warn(netdev, - "%s, failed to create ipv4 udp matcher, err: %d\n", - INIT_ERR_PREFIX, err); - goto err_matcher_ipv4_udp; + tbl = nat ? fs_smfs->ct_nat_tbl : fs_smfs->ct_tbl; + dr_matcher = mlx5_ct_fs_smfs_matcher_create(fs, tbl, ipv4, tcp, prio); + if (IS_ERR(dr_matcher)) { + netdev_warn(fs->netdev, + "ct_fs_smfs: failed to create matcher (nat %d, ipv4 %d, tcp %d), err: %ld\n", + nat, ipv4, tcp, PTR_ERR(dr_matcher)); + + smfs_matcher = ERR_CAST(dr_matcher); + goto out_unlock; } - ++prio; - ct_matchers->ipv6_tcp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, false, true, prio); - if (IS_ERR(ct_matchers->ipv6_tcp)) { - err = PTR_ERR(ct_matchers->ipv6_tcp); - netdev_warn(netdev, - "%s, failed to create ipv6 tcp matcher, err: %d\n", - INIT_ERR_PREFIX, err); - goto err_matcher_ipv6_tcp; - } + smfs_matcher->dr_matcher = dr_matcher; + smfs_matcher->prio = prio; + list_add(&smfs_matcher->list, prev); + refcount_set(&smfs_matcher->ref, 1); - ++prio; - ct_matchers->ipv6_udp = mlx5_ct_fs_smfs_matcher_create(fs, tbl, false, false, prio); - if (IS_ERR(ct_matchers->ipv6_udp)) { - err = PTR_ERR(ct_matchers->ipv6_udp); - netdev_warn(netdev, - "%s, failed to create ipv6 tcp matcher, err: %d\n", - INIT_ERR_PREFIX, err); - goto err_matcher_ipv6_udp; - } - - return 0; - -err_matcher_ipv6_udp: - mlx5_smfs_matcher_destroy(ct_matchers->ipv6_tcp); -err_matcher_ipv6_tcp: - mlx5_smfs_matcher_destroy(ct_matchers->ipv4_udp); -err_matcher_ipv4_udp: - mlx5_smfs_matcher_destroy(ct_matchers->ipv4_tcp); - return 0; +out_unlock: + mutex_unlock(&fs_smfs->lock); + return smfs_matcher; } static void -mlx5_ct_fs_smfs_matchers_destroy(struct mlx5_ct_fs_smfs_matchers *ct_matchers) +mlx5_ct_fs_smfs_matcher_put(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_smfs_matcher *smfs_matcher) { - mlx5_smfs_matcher_destroy(ct_matchers->ipv6_udp); - mlx5_smfs_matcher_destroy(ct_matchers->ipv6_tcp); - mlx5_smfs_matcher_destroy(ct_matchers->ipv4_udp); - mlx5_smfs_matcher_destroy(ct_matchers->ipv4_tcp); + struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); + + if (!refcount_dec_and_mutex_lock(&smfs_matcher->ref, &fs_smfs->lock)) + return; + + mlx5_smfs_matcher_destroy(smfs_matcher->dr_matcher); + list_del(&smfs_matcher->list); + mutex_unlock(&fs_smfs->lock); } static int @@ -170,7 +184,6 @@ mlx5_ct_fs_smfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct, { struct mlx5dr_table *ct_tbl, *ct_nat_tbl, *post_ct_tbl; struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); - int err; post_ct_tbl = mlx5_smfs_table_get_from_fs_ft(post_ct); ct_nat_tbl = mlx5_smfs_table_get_from_fs_ft(ct_nat); @@ -184,28 +197,18 @@ mlx5_ct_fs_smfs_init(struct mlx5_ct_fs *fs, struct mlx5_flow_table *ct, ct_dbg("using smfs steering"); - err = mlx5_ct_fs_smfs_matchers_create(fs, ct_tbl, &fs_smfs->ct_matchers); - if (err) - goto err_init; - - err = mlx5_ct_fs_smfs_matchers_create(fs, ct_nat_tbl, &fs_smfs->ct_matchers_nat); - if (err) - goto err_matchers_nat; - fs_smfs->fwd_action = mlx5_smfs_action_create_dest_table(post_ct_tbl); if (!fs_smfs->fwd_action) { - err = -EINVAL; - goto err_action_create; + return -EINVAL; } - return 0; + fs_smfs->ct_tbl = ct_tbl; + fs_smfs->ct_nat_tbl = ct_nat_tbl; + mutex_init(&fs_smfs->lock); + INIT_LIST_HEAD(&fs_smfs->matchers.used); + INIT_LIST_HEAD(&fs_smfs->matchers_nat.used); -err_action_create: - mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers_nat); -err_matchers_nat: - mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers); -err_init: - return err; + return 0; } static void @@ -214,8 +217,6 @@ mlx5_ct_fs_smfs_destroy(struct mlx5_ct_fs *fs) struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); mlx5_smfs_action_destroy(fs_smfs->fwd_action); - mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers_nat); - mlx5_ct_fs_smfs_matchers_destroy(&fs_smfs->ct_matchers); } static inline bool @@ -285,10 +286,9 @@ mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr, struct flow_rule *flow_rule) { struct mlx5_ct_fs_smfs *fs_smfs = mlx5_ct_fs_priv(fs); - struct mlx5_ct_fs_smfs_matchers *matchers; + struct mlx5_ct_fs_smfs_matcher *smfs_matcher; struct mlx5_ct_fs_smfs_rule *smfs_rule; struct mlx5dr_action *actions[5]; - struct mlx5dr_matcher *matcher; struct mlx5dr_rule *rule; int num_actions = 0, err; bool nat, tcp, ipv4; @@ -315,22 +315,27 @@ mlx5_ct_fs_smfs_ct_rule_add(struct mlx5_ct_fs *fs, struct mlx5_flow_spec *spec, tcp = MLX5_GET(fte_match_param, spec->match_value, outer_headers.ip_protocol) == IPPROTO_TCP; - matchers = nat ? &fs_smfs->ct_matchers_nat : &fs_smfs->ct_matchers; - matcher = ipv4 ? (tcp ? matchers->ipv4_tcp : matchers->ipv4_udp) : - (tcp ? matchers->ipv6_tcp : matchers->ipv6_udp); + smfs_matcher = mlx5_ct_fs_smfs_matcher_get(fs, nat, ipv4, tcp); + if (IS_ERR(smfs_matcher)) { + err = PTR_ERR(smfs_matcher); + goto err_matcher; + } - rule = mlx5_smfs_rule_create(matcher, spec, num_actions, actions, + rule = mlx5_smfs_rule_create(smfs_matcher->dr_matcher, spec, num_actions, actions, MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT); if (!rule) { err = -EINVAL; - goto err_rule; + goto err_create; } smfs_rule->rule = rule; + smfs_rule->smfs_matcher = smfs_matcher; return &smfs_rule->fs_rule; -err_rule: +err_create: + mlx5_ct_fs_smfs_matcher_put(fs, smfs_matcher); +err_matcher: mlx5_smfs_action_destroy(smfs_rule->count_action); err_count: kfree(smfs_rule); @@ -345,6 +350,7 @@ mlx5_ct_fs_smfs_ct_rule_del(struct mlx5_ct_fs *fs, struct mlx5_ct_fs_rule *fs_ru fs_rule); mlx5_smfs_rule_destroy(smfs_rule->rule); + mlx5_ct_fs_smfs_matcher_put(fs, smfs_rule->smfs_matcher); mlx5_smfs_action_destroy(smfs_rule->count_action); kfree(smfs_rule); } From 271907ee2f29cd1078fd219f0778fd824fb1971c Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Mon, 17 Jan 2022 15:14:44 +0200 Subject: [PATCH 13/15] net/mlx5: Query the maximum MCIA register read size from firmware The MCIA register supports either 12 or 32 dwords, use the correct value by querying the capability from the MCAM register. Signed-off-by: Gal Pressman Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/port.c | 8 +++++++- include/linux/mlx5/mlx5_ifc.h | 4 +++- include/linux/mlx5/port.h | 1 - 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 289b29a23418..418ab777f6e8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -365,6 +365,12 @@ static void mlx5_sfp_eeprom_params_set(u16 *i2c_addr, int *page_num, u16 *offset *offset -= MLX5_EEPROM_PAGE_LENGTH; } +static int mlx5_mcia_max_bytes(struct mlx5_core_dev *dev) +{ + /* mcia supports either 12 dwords or 32 dwords */ + return (MLX5_CAP_MCAM_FEATURE(dev, mcia_32dwords) ? 32 : 12) * sizeof(u32); +} + static int mlx5_query_mcia(struct mlx5_core_dev *dev, struct mlx5_module_eeprom_query_params *params, u8 *data) { @@ -374,7 +380,7 @@ static int mlx5_query_mcia(struct mlx5_core_dev *dev, void *ptr; u16 size; - size = min_t(int, params->size, MLX5_EEPROM_MAX_BYTES); + size = min_t(int, params->size, mlx5_mcia_max_bytes(dev)); MLX5_SET(mcia_reg, in, l, 0); MLX5_SET(mcia_reg, in, size, size); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 318fae4b3560..745107ff681d 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -9691,7 +9691,9 @@ struct mlx5_ifc_pcam_reg_bits { }; struct mlx5_ifc_mcam_enhanced_features_bits { - u8 reserved_at_0[0x6a]; + u8 reserved_at_0[0x5d]; + u8 mcia_32dwords[0x1]; + u8 reserved_at_5e[0xc]; u8 reset_state[0x1]; u8 ptpcyc2realtime_modify[0x1]; u8 reserved_at_6c[0x2]; diff --git a/include/linux/mlx5/port.h b/include/linux/mlx5/port.h index 77ea4f9c5265..402413b3e914 100644 --- a/include/linux/mlx5/port.h +++ b/include/linux/mlx5/port.h @@ -56,7 +56,6 @@ enum mlx5_an_status { MLX5_AN_LINK_DOWN = 4, }; -#define MLX5_EEPROM_MAX_BYTES 32 #define MLX5_EEPROM_IDENTIFIER_BYTE_MASK 0x000000ff #define MLX5_I2C_ADDR_LOW 0x50 #define MLX5_I2C_ADDR_HIGH 0x51 From fcb610a86c53dfcfbb2aa62e704481112752f367 Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Mon, 17 Jan 2022 15:53:06 +0200 Subject: [PATCH 14/15] net/mlx5: Parse module mapping using mlx5_ifc The assumption that the first byte in the module mapping dword is the module number shouldn't be hard-coded in the driver, but come from mlx5_ifc structs. While at it, fix the incorrect width for the 'rx_lane' and 'tx_lane' fields. Signed-off-by: Gal Pressman Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/port.c | 6 +++--- include/linux/mlx5/mlx5_ifc.h | 8 ++++---- include/linux/mlx5/port.h | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 418ab777f6e8..493cacb4610b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -275,7 +275,6 @@ static int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num) { u32 in[MLX5_ST_SZ_DW(pmlp_reg)] = {0}; u32 out[MLX5_ST_SZ_DW(pmlp_reg)]; - int module_mapping; int err; MLX5_SET(pmlp_reg, in, local_port, 1); @@ -284,8 +283,9 @@ static int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num) if (err) return err; - module_mapping = MLX5_GET(pmlp_reg, out, lane0_module_mapping); - *module_num = module_mapping & MLX5_EEPROM_IDENTIFIER_BYTE_MASK; + *module_num = MLX5_GET(lane_2_module_mapping, + MLX5_ADDR_OF(pmlp_reg, out, lane0_module_mapping), + module); return 0; } diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 745107ff681d..91b7f730ed91 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -9888,10 +9888,10 @@ struct mlx5_ifc_pcmr_reg_bits { }; struct mlx5_ifc_lane_2_module_mapping_bits { - u8 reserved_at_0[0x6]; - u8 rx_lane[0x2]; - u8 reserved_at_8[0x6]; - u8 tx_lane[0x2]; + u8 reserved_at_0[0x4]; + u8 rx_lane[0x4]; + u8 reserved_at_8[0x4]; + u8 tx_lane[0x4]; u8 reserved_at_10[0x8]; u8 module[0x8]; }; diff --git a/include/linux/mlx5/port.h b/include/linux/mlx5/port.h index 402413b3e914..28a928b0684b 100644 --- a/include/linux/mlx5/port.h +++ b/include/linux/mlx5/port.h @@ -56,7 +56,6 @@ enum mlx5_an_status { MLX5_AN_LINK_DOWN = 4, }; -#define MLX5_EEPROM_IDENTIFIER_BYTE_MASK 0x000000ff #define MLX5_I2C_ADDR_LOW 0x50 #define MLX5_I2C_ADDR_HIGH 0x51 #define MLX5_EEPROM_PAGE_LENGTH 256 From 970adfb76095fa719778d70a6b86030d2feb88dd Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Wed, 26 Jan 2022 16:28:23 +0200 Subject: [PATCH 15/15] net/mlx5e: Remove overzealous validations in netlink EEPROM query Unlike the legacy EEPROM callbacks, when using the netlink EEPROM query (get_module_eeprom_by_page) the driver should not try to validate the query parameters, but just perform the read requested by the userspace. Recent discussion in the mailing list: https://lore.kernel.org/netdev/20220120093051.70845141@kicinski-fedora-PC1C0HJN.hsd1.ca.comcast.net/ Signed-off-by: Gal Pressman Reviewed-by: Ido Schimmel Reviewed-by: Maxim Mikityanskiy Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/port.c | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 493cacb4610b..e1bd54574ea5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -451,35 +451,12 @@ int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev, struct mlx5_module_eeprom_query_params *params, u8 *data) { - u8 module_id; int err; err = mlx5_query_module_num(dev, ¶ms->module_number); if (err) return err; - err = mlx5_query_module_id(dev, params->module_number, &module_id); - if (err) - return err; - - switch (module_id) { - case MLX5_MODULE_ID_SFP: - if (params->page > 0) - return -EINVAL; - break; - case MLX5_MODULE_ID_QSFP: - case MLX5_MODULE_ID_QSFP28: - case MLX5_MODULE_ID_QSFP_PLUS: - if (params->page > 3) - return -EINVAL; - break; - case MLX5_MODULE_ID_DSFP: - break; - default: - mlx5_core_err(dev, "Module ID not recognized: 0x%x\n", module_id); - return -EINVAL; - } - if (params->i2c_address != MLX5_I2C_ADDR_HIGH && params->i2c_address != MLX5_I2C_ADDR_LOW) { mlx5_core_err(dev, "I2C address not recognized: 0x%x\n", params->i2c_address);