From 3df54c870f52b4c47b53eead8d22a109f741b91c Mon Sep 17 00:00:00 2001 From: Kelvin Cao Date: Wed, 29 Nov 2017 10:55:24 -0700 Subject: ntb_hw_switchtec: Allow using Switchtec NTB in multi-partition setups Allow using Switchtec NTB in setups that have more than two partitions. Note: this does not enable having multi-host communication, it only allows for a single NTB link between two hosts in a network that might have more than two. Use following logic to determine the NT peer partition: 1) If there are 2 partitions, and the target vector is set in the Switchtec configuration, use the partition specified in target vector. 2) If there are 2 partitions and target vector is unset use the only other partition as specified in the NT EP map. 3) If there are more than 2 partitions and target vector is set use the other partition specified in target vector. 4) If there are more than 2 partitions and target vector is unset, this is invalid and report an error. Signed-off-by: Kelvin Cao [logang@deltatee.com: commit message fleshed out] Signed-off-by: Logan Gunthorpe Reviewed-by: Logan Gunthorpe Signed-off-by: Jon Mason --- include/linux/switchtec.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index 09d73d0d1aa8..d4a7c18b42cf 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -168,6 +168,14 @@ struct ntb_info_regs { u16 reserved1; u64 ep_map; u16 requester_id; + u16 reserved2; + u32 reserved3[4]; + struct nt_partition_info { + u32 xlink_enabled; + u32 target_part_low; + u32 target_part_high; + u32 reserved; + } ntp_info[48]; } __packed; struct part_cfg_regs { -- cgit v1.2.3 From 45f447deb29081a7df7b81f2cd9cc8121994d988 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Wed, 29 Nov 2017 10:55:28 -0700 Subject: ntb_hw_switchtec: Expand PFF CSR registers The PFF CSR registers actual mirrors the PCI configuration space for all the ports in the switch. Previously, this was not needed by the driver but will be used by the crosslink code to enumerate the bus in an host-less centre partition. Signed-off-by: Logan Gunthorpe Signed-off-by: Jon Mason --- include/linux/switchtec.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index d4a7c18b42cf..6d325a7a0c19 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -292,7 +292,20 @@ enum { struct pff_csr_regs { u16 vendor_id; u16 device_id; - u32 pci_cfg_header[15]; + u16 pcicmd; + u16 pcists; + u32 pci_class; + u32 pci_opts; + union { + u32 pci_bar[6]; + u64 pci_bar64[3]; + }; + u32 pci_cardbus; + u32 pci_subsystem_id; + u32 pci_expansion_rom; + u32 pci_cap_ptr; + u32 reserved1; + u32 pci_irq; u32 pci_cap_region[48]; u32 pcie_cap_region[448]; u32 indirect_gas_window[128]; -- cgit v1.2.3 From 01752501820277d217a7b52548d9c948f98d2c56 Mon Sep 17 00:00:00 2001 From: Logan Gunthorpe Date: Wed, 29 Nov 2017 10:55:29 -0700 Subject: ntb_hw_switchtec: Add initialization code for crosslink Crosslink is a feature of the Switchtec switches that is similar to the B2B mode of other NTB devices. It allows a system to be designed that is perfectly symmetric with two identical switches that link two hosts together. In order for the system to be symmetric, there is an empty host-less partition between the two switches which the host must enumerate and assign BAR addresses to. The firmware in the switch manages this specially so that the BAR addresses on both sides of the empty partition will be identical despite being in the same partition with the same address space. The driver determines whether crosslink is enabled by a flag set in the NTB partition info registers which are set by the switch's configuration file. When crosslink is enabled, a reserved LUT window is setup to point to the peer's switch's NTB registers and the local MWs are set to forward to the host-less partition's BARs. (Yes, this hurts my brain too.) Once this is setup, largely the same NTB infrastructure is used to communicate between the two hosts. Signed-off-by: Logan Gunthorpe Signed-off-by: Jon Mason --- include/linux/ntb.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/linux/ntb.h b/include/linux/ntb.h index c308964777eb..ea3be7275a5e 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -71,6 +71,7 @@ struct pci_dev; * @NTB_TOPO_B2B_USD: On primary side of local ntb upstream of remote ntb. * @NTB_TOPO_B2B_DSD: On primary side of local ntb downstream of remote ntb. * @NTB_TOPO_SWITCH: Connected via a switch which supports ntb. + * @NTB_TOPO_CROSSLINK: Connected via two symmetric switchecs */ enum ntb_topo { NTB_TOPO_NONE = -1, @@ -79,6 +80,7 @@ enum ntb_topo { NTB_TOPO_B2B_USD, NTB_TOPO_B2B_DSD, NTB_TOPO_SWITCH, + NTB_TOPO_CROSSLINK, }; static inline int ntb_topo_is_b2b(enum ntb_topo topo) @@ -94,12 +96,13 @@ static inline int ntb_topo_is_b2b(enum ntb_topo topo) static inline char *ntb_topo_string(enum ntb_topo topo) { switch (topo) { - case NTB_TOPO_NONE: return "NTB_TOPO_NONE"; - case NTB_TOPO_PRI: return "NTB_TOPO_PRI"; - case NTB_TOPO_SEC: return "NTB_TOPO_SEC"; - case NTB_TOPO_B2B_USD: return "NTB_TOPO_B2B_USD"; - case NTB_TOPO_B2B_DSD: return "NTB_TOPO_B2B_DSD"; - case NTB_TOPO_SWITCH: return "NTB_TOPO_SWITCH"; + case NTB_TOPO_NONE: return "NTB_TOPO_NONE"; + case NTB_TOPO_PRI: return "NTB_TOPO_PRI"; + case NTB_TOPO_SEC: return "NTB_TOPO_SEC"; + case NTB_TOPO_B2B_USD: return "NTB_TOPO_B2B_USD"; + case NTB_TOPO_B2B_DSD: return "NTB_TOPO_B2B_DSD"; + case NTB_TOPO_SWITCH: return "NTB_TOPO_SWITCH"; + case NTB_TOPO_CROSSLINK: return "NTB_TOPO_CROSSLINK"; } return "NTB_TOPO_INVALID"; } -- cgit v1.2.3 From b87ab21935d76922362ff98a5a78f16e2e956ead Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 6 Dec 2017 17:31:52 +0300 Subject: NTB: Rename NTB messaging API methods There is a common methods signature form used over all the NTB API like functions naming scheme, arguments names and order, etc. Recently added NTB messaging API IO callbacks were named a bit different so should be renamed to be in compliance with the rest of the API. Signed-off-by: Serge Semin Signed-off-by: Jon Mason --- include/linux/ntb.h | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/include/linux/ntb.h b/include/linux/ntb.h index ea3be7275a5e..21b1826a0953 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -253,7 +253,7 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops) * @msg_set_mask: See ntb_msg_set_mask(). * @msg_clear_mask: See ntb_msg_clear_mask(). * @msg_read: See ntb_msg_read(). - * @msg_write: See ntb_msg_write(). + * @peer_msg_write: See ntb_peer_msg_write(). */ struct ntb_dev_ops { int (*port_number)(struct ntb_dev *ntb); @@ -324,8 +324,8 @@ struct ntb_dev_ops { int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits); int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits); int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits); - int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg); - int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg); + u32 (*msg_read)(struct ntb_dev *ntb, int *pidx, int midx); + int (*peer_msg_write)(struct ntb_dev *ntb, int pidx, int midx, u32 msg); }; static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) @@ -387,7 +387,7 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) /* !ops->msg_set_mask == !ops->msg_count && */ /* !ops->msg_clear_mask == !ops->msg_count && */ !ops->msg_read == !ops->msg_count && - !ops->msg_write == !ops->msg_count && + !ops->peer_msg_write == !ops->msg_count && 1; } @@ -1462,31 +1462,29 @@ static inline int ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits) } /** - * ntb_msg_read() - read message register with specified index + * ntb_msg_read() - read inbound message register with specified index * @ntb: NTB device context. - * @midx: Message register index * @pidx: OUT - Port index of peer device a message retrieved from - * @msg: OUT - Data + * @midx: Message register index * * Read data from the specified message register. Source port index of a * message is retrieved as well. * - * Return: Zero on success, otherwise a negative error number. + * Return: The value of the inbound message register. */ -static inline int ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, - u32 *msg) +static inline u32 ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx) { if (!ntb->ops->msg_read) - return -EINVAL; + return ~(u32)0; - return ntb->ops->msg_read(ntb, midx, pidx, msg); + return ntb->ops->msg_read(ntb, pidx, midx); } /** - * ntb_msg_write() - write data to the specified message register + * ntb_peer_msg_write() - write data to the specified peer message register * @ntb: NTB device context. - * @midx: Message register index * @pidx: Port index of peer device a message being sent to + * @midx: Message register index * @msg: Data to send * * Send data to a specified peer device using the defined message register. @@ -1495,13 +1493,13 @@ static inline int ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, * * Return: Zero on success, otherwise a negative error number. */ -static inline int ntb_msg_write(struct ntb_dev *ntb, int midx, int pidx, - u32 msg) +static inline int ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx, + u32 msg) { - if (!ntb->ops->msg_write) + if (!ntb->ops->peer_msg_write) return -EINVAL; - return ntb->ops->msg_write(ntb, midx, pidx, msg); + return ntb->ops->peer_msg_write(ntb, pidx, midx, msg); } #endif -- cgit v1.2.3 From f1678a4c66a5c2cb43f744f7dc7e048d59690166 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 6 Dec 2017 17:31:54 +0300 Subject: NTB: Fix UB/bug in ntb_mw_get_align() Simple (1 << pidx) operation causes undefined behaviour when pidx >= 32. It must be casted to u64 to match the actual return value of ntb_link_is_up() method, so to have all the possible peer indexes covered and to get rid of undefined behaviour. Additionally there are special macros in "linux/bitops.h" to perform the bit-set-shift operations, so it's recommended to have them used for proper bit setting. Signed-off-by: Serge Semin Reviewed-by: Logan Gunthorpe Signed-off-by: Jon Mason --- include/linux/ntb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/ntb.h b/include/linux/ntb.h index 21b1826a0953..181d16601dd9 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -767,7 +767,7 @@ static inline int ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx, resource_size_t *size_align, resource_size_t *size_max) { - if (!(ntb_link_is_up(ntb, NULL, NULL) & (1 << pidx))) + if (!(ntb_link_is_up(ntb, NULL, NULL) & BIT_ULL(pidx))) return -ENOTCONN; return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align, -- cgit v1.2.3