diff options
Diffstat (limited to 'drivers/ntb/hw/amd')
-rw-r--r-- | drivers/ntb/hw/amd/ntb_hw_amd.c | 290 | ||||
-rw-r--r-- | drivers/ntb/hw/amd/ntb_hw_amd.h | 8 |
2 files changed, 247 insertions, 51 deletions
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c index e52b300b2f5b..9e310e1ad4d0 100644 --- a/drivers/ntb/hw/amd/ntb_hw_amd.c +++ b/drivers/ntb/hw/amd/ntb_hw_amd.c @@ -195,26 +195,117 @@ static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return 0; } -static int amd_link_is_up(struct amd_ntb_dev *ndev) +static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev) { - if (!ndev->peer_sta) - return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); + struct pci_dev *pdev = NULL; + struct pci_dev *pci_swds = NULL; + struct pci_dev *pci_swus = NULL; + u32 stat; + int rc; - if (ndev->peer_sta & AMD_LINK_UP_EVENT) { - ndev->peer_sta = 0; - return 1; + if (ndev->ntb.topo == NTB_TOPO_SEC) { + /* Locate the pointer to Downstream Switch for this device */ + pci_swds = pci_upstream_bridge(ndev->ntb.pdev); + if (pci_swds) { + /* + * Locate the pointer to Upstream Switch for + * the Downstream Switch. + */ + pci_swus = pci_upstream_bridge(pci_swds); + if (pci_swus) { + rc = pcie_capability_read_dword(pci_swus, + PCI_EXP_LNKCTL, + &stat); + if (rc) + return 0; + } else { + return 0; + } + } else { + return 0; + } + } else if (ndev->ntb.topo == NTB_TOPO_PRI) { + /* + * For NTB primary, we simply read the Link Status and control + * register of the NTB device itself. + */ + pdev = ndev->ntb.pdev; + rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat); + if (rc) + return 0; + } else { + /* Catch all for everything else */ + return 0; } - /* If peer_sta is reset or D0 event, the ISR has - * started a timer to check link status of hardware. - * So here just clear status bit. And if peer_sta is - * D3 or PME_TO, D0/reset event will be happened when - * system wakeup/poweron, so do nothing here. + ndev->lnk_sta = stat; + + return 1; +} + +static int amd_link_is_up(struct amd_ntb_dev *ndev) +{ + int ret; + + /* + * We consider the link to be up under two conditions: + * + * - When a link-up event is received. This is indicated by + * AMD_LINK_UP_EVENT set in peer_sta. + * - When driver on both sides of the link have been loaded. + * This is indicated by bit 1 being set in the peer + * SIDEINFO register. + * + * This function should return 1 when the latter of the above + * two conditions is true. + * + * Now consider the sequence of events - Link-Up event occurs, + * then the peer side driver loads. In this case, we would have + * received LINK_UP event and bit 1 of peer SIDEINFO is also + * set. What happens now if the link goes down? Bit 1 of + * peer SIDEINFO remains set, but LINK_DOWN bit is set in + * peer_sta. So we should return 0 from this function. Not only + * that, we clear bit 1 of peer SIDEINFO to 0, since the peer + * side driver did not even get a chance to clear it before + * the link went down. This can be the case of surprise link + * removal. + * + * LINK_UP event will always occur before the peer side driver + * gets loaded the very first time. So there can be a case when + * the LINK_UP event has occurred, but the peer side driver hasn't + * yet loaded. We return 0 in that case. + * + * There is also a special case when the primary side driver is + * unloaded and then loaded again. Since there is no change in + * the status of NTB secondary in this case, there is no Link-Up + * or Link-Down notification received. We recognize this condition + * with peer_sta being set to 0. + * + * If bit 1 of peer SIDEINFO register is not set, then we + * simply return 0 irrespective of the link up or down status + * set in peer_sta. */ - if (ndev->peer_sta & AMD_PEER_RESET_EVENT) - ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; - else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT)) - ndev->peer_sta = 0; + ret = amd_poll_link(ndev); + if (ret) { + /* + * We need to check the below only for NTB primary. For NTB + * secondary, simply checking the result of PSIDE_INFO + * register will suffice. + */ + if (ndev->ntb.topo == NTB_TOPO_PRI) { + if ((ndev->peer_sta & AMD_LINK_UP_EVENT) || + (ndev->peer_sta == 0)) + return ret; + else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) { + /* Clear peer sideinfo register */ + amd_clear_side_info_reg(ndev, true); + + return 0; + } + } else { /* NTB_TOPO_SEC */ + return ret; + } + } return 0; } @@ -253,7 +344,6 @@ static int amd_ntb_link_enable(struct ntb_dev *ntb, { struct amd_ntb_dev *ndev = ntb_ndev(ntb); void __iomem *mmio = ndev->self_mmio; - u32 ntb_ctl; /* Enable event interrupt */ ndev->int_mask &= ~AMD_EVENT_INTMASK; @@ -263,10 +353,6 @@ static int amd_ntb_link_enable(struct ntb_dev *ntb, return -EINVAL; dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); - ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); - ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); - writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); - return 0; } @@ -274,7 +360,6 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb) { struct amd_ntb_dev *ndev = ntb_ndev(ntb); void __iomem *mmio = ndev->self_mmio; - u32 ntb_ctl; /* Disable event interrupt */ ndev->int_mask |= AMD_EVENT_INTMASK; @@ -284,10 +369,6 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb) return -EINVAL; dev_dbg(&ntb->pdev->dev, "Enabling Link.\n"); - ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); - ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); - writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); - return 0; } @@ -493,8 +574,6 @@ static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) reg = readl(mmio + AMD_SMUACK_OFFSET); reg |= bit; writel(reg, mmio + AMD_SMUACK_OFFSET); - - ndev->peer_sta |= bit; } static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) @@ -512,10 +591,16 @@ static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) status &= AMD_EVENT_INTMASK; switch (status) { case AMD_PEER_FLUSH_EVENT: + ndev->peer_sta |= AMD_PEER_FLUSH_EVENT; dev_info(dev, "Flush is done.\n"); break; case AMD_PEER_RESET_EVENT: - amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); + case AMD_LINK_DOWN_EVENT: + ndev->peer_sta |= status; + if (status == AMD_LINK_DOWN_EVENT) + ndev->peer_sta &= ~AMD_LINK_UP_EVENT; + + amd_ack_smu(ndev, status); /* link down first */ ntb_link_event(&ndev->ntb); @@ -526,7 +611,12 @@ static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) case AMD_PEER_D3_EVENT: case AMD_PEER_PMETO_EVENT: case AMD_LINK_UP_EVENT: - case AMD_LINK_DOWN_EVENT: + ndev->peer_sta |= status; + if (status == AMD_LINK_UP_EVENT) + ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT; + else if (status == AMD_PEER_D3_EVENT) + ndev->peer_sta &= ~AMD_PEER_D0_EVENT; + amd_ack_smu(ndev, status); /* link down */ @@ -540,6 +630,8 @@ static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) if (status & 0x1) dev_info(dev, "Wakeup is done.\n"); + ndev->peer_sta |= AMD_PEER_D0_EVENT; + ndev->peer_sta &= ~AMD_PEER_D3_EVENT; amd_ack_smu(ndev, AMD_PEER_D0_EVENT); /* start a timer to poll link status */ @@ -550,6 +642,39 @@ static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) dev_info(dev, "event status = 0x%x.\n", status); break; } + + /* Clear the interrupt status */ + writel(status, mmio + AMD_INTSTAT_OFFSET); +} + +static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec) +{ + struct device *dev = &ndev->ntb.pdev->dev; + u64 status; + + status = amd_ntb_db_read(&ndev->ntb); + + dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec); + + /* + * Since we had reserved highest order bit of DB for signaling peer of + * a special event, this is the only status bit we should be concerned + * here now. + */ + if (status & BIT(ndev->db_last_bit)) { + ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit)); + /* send link down event notification */ + ntb_link_event(&ndev->ntb); + + /* + * If we are here, that means the peer has signalled a special + * event which notifies that the peer driver has been + * un-loaded for some reason. Since there is a chance that the + * peer will load its driver again sometime, we schedule link + * polling routine. + */ + schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); + } } static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) @@ -559,8 +684,10 @@ static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) amd_handle_event(ndev, vec); - if (vec < AMD_DB_CNT) + if (vec < AMD_DB_CNT) { + amd_handle_db_event(ndev, vec); ntb_db_event(&ndev->ntb, vec); + } return IRQ_HANDLED; } @@ -842,26 +969,18 @@ static inline void ndev_init_struct(struct amd_ntb_dev *ndev, static int amd_poll_link(struct amd_ntb_dev *ndev) { void __iomem *mmio = ndev->peer_mmio; - u32 reg, stat; - int rc; + u32 reg; reg = readl(mmio + AMD_SIDEINFO_OFFSET); - reg &= NTB_LIN_STA_ACTIVE_BIT; + reg &= AMD_SIDE_READY; dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg); - if (reg == ndev->cntl_sta) - return 0; - ndev->cntl_sta = reg; - rc = pci_read_config_dword(ndev->ntb.pdev, - AMD_LINK_STATUS_OFFSET, &stat); - if (rc) - return 0; - ndev->lnk_sta = stat; + amd_ntb_get_link_status(ndev); - return 1; + return ndev->cntl_sta; } static void amd_link_hb(struct work_struct *work) @@ -880,11 +999,16 @@ static int amd_init_isr(struct amd_ntb_dev *ndev) return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); } -static void amd_init_side_info(struct amd_ntb_dev *ndev) +static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer) { - void __iomem *mmio = ndev->self_mmio; + void __iomem *mmio = NULL; unsigned int reg; + if (peer) + mmio = ndev->peer_mmio; + else + mmio = ndev->self_mmio; + reg = readl(mmio + AMD_SIDEINFO_OFFSET); if (!(reg & AMD_SIDE_READY)) { reg |= AMD_SIDE_READY; @@ -892,11 +1016,16 @@ static void amd_init_side_info(struct amd_ntb_dev *ndev) } } -static void amd_deinit_side_info(struct amd_ntb_dev *ndev) +static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer) { - void __iomem *mmio = ndev->self_mmio; + void __iomem *mmio = NULL; unsigned int reg; + if (peer) + mmio = ndev->peer_mmio; + else + mmio = ndev->self_mmio; + reg = readl(mmio + AMD_SIDEINFO_OFFSET); if (reg & AMD_SIDE_READY) { reg &= ~AMD_SIDE_READY; @@ -905,6 +1034,30 @@ static void amd_deinit_side_info(struct amd_ntb_dev *ndev) } } +static void amd_init_side_info(struct amd_ntb_dev *ndev) +{ + void __iomem *mmio = ndev->self_mmio; + u32 ntb_ctl; + + amd_set_side_info_reg(ndev, false); + + ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); + ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); + writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); +} + +static void amd_deinit_side_info(struct amd_ntb_dev *ndev) +{ + void __iomem *mmio = ndev->self_mmio; + u32 ntb_ctl; + + amd_clear_side_info_reg(ndev, false); + + ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); + ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); + writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); +} + static int amd_init_ntb(struct amd_ntb_dev *ndev) { void __iomem *mmio = ndev->self_mmio; @@ -935,8 +1088,6 @@ static int amd_init_ntb(struct amd_ntb_dev *ndev) return -EINVAL; } - ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; - /* Mask event interrupts */ writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); @@ -957,6 +1108,7 @@ static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) static int amd_init_dev(struct amd_ntb_dev *ndev) { + void __iomem *mmio = ndev->self_mmio; struct pci_dev *pdev; int rc = 0; @@ -977,6 +1129,25 @@ static int amd_init_dev(struct amd_ntb_dev *ndev) } ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + /* + * We reserve the highest order bit of the DB register which will + * be used to notify peer when the driver on this side is being + * un-loaded. + */ + ndev->db_last_bit = + find_last_bit((unsigned long *)&ndev->db_valid_mask, + hweight64(ndev->db_valid_mask)); + writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET); + /* + * Since now there is one less bit to account for, the DB count + * and DB mask should be adjusted accordingly. + */ + ndev->db_count -= 1; + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + + /* Enable Link-Up and Link-Down event interrupts */ + ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT); + writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); return 0; } @@ -1111,9 +1282,31 @@ static void amd_ntb_pci_remove(struct pci_dev *pdev) { struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); + /* + * Clear the READY bit in SIDEINFO register before sending DB event + * to the peer. This will make sure that when the peer handles the + * DB event, it correctly reads this bit as being 0. + */ + amd_deinit_side_info(ndev); + ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit)); ntb_unregister_device(&ndev->ntb); ndev_deinit_debugfs(ndev); + amd_deinit_dev(ndev); + amd_ntb_deinit_pci(ndev); + kfree(ndev); +} + +static void amd_ntb_pci_shutdown(struct pci_dev *pdev) +{ + struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); + + /* Send link down notification */ + ntb_link_event(&ndev->ntb); + amd_deinit_side_info(ndev); + ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit)); + ntb_unregister_device(&ndev->ntb); + ndev_deinit_debugfs(ndev); amd_deinit_dev(ndev); amd_ntb_deinit_pci(ndev); kfree(ndev); @@ -1149,6 +1342,7 @@ static struct pci_driver amd_ntb_pci_driver = { .id_table = amd_ntb_pci_tbl, .probe = amd_ntb_pci_probe, .remove = amd_ntb_pci_remove, + .shutdown = amd_ntb_pci_shutdown, }; static int __init amd_ntb_pci_driver_init(void) diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.h b/drivers/ntb/hw/amd/ntb_hw_amd.h index 139a307147bc..5f337b1572a0 100644 --- a/drivers/ntb/hw/amd/ntb_hw_amd.h +++ b/drivers/ntb/hw/amd/ntb_hw_amd.h @@ -53,11 +53,8 @@ #include <linux/pci.h> #define AMD_LINK_HB_TIMEOUT msecs_to_jiffies(1000) -#define AMD_LINK_STATUS_OFFSET 0x68 -#define NTB_LIN_STA_ACTIVE_BIT 0x00000002 #define NTB_LNK_STA_SPEED_MASK 0x000F0000 #define NTB_LNK_STA_WIDTH_MASK 0x03F00000 -#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT)) #define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16) #define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20) @@ -196,6 +193,7 @@ struct amd_ntb_dev { u64 db_valid_mask; u64 db_mask; + u64 db_last_bit; u32 int_mask; struct msix_entry *msix; @@ -218,4 +216,8 @@ struct amd_ntb_dev { #define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb) #define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work) +static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer); +static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer); +static int amd_poll_link(struct amd_ntb_dev *ndev); + #endif |