From 69054666df0a9b4e8331319f98b6b9a88bc3fcc4 Mon Sep 17 00:00:00 2001 From: Sagiv Ozeri Date: Mon, 20 May 2019 12:33:20 +0300 Subject: RDMA/qedr: Fix incorrect device rate. Use the correct enum value introduced in commit 12113a35ada6 ("IB/core: Add HDR speed enum") Prior to this change a 50Gbps port would show 40Gbps. This patch also cleaned up the redundant redefiniton of ib speeds for qedr. Fixes: 12113a35ada6 ("IB/core: Add HDR speed enum") Signed-off-by: Sagiv Ozeri Signed-off-by: Michal Kalderon Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/qedr/verbs.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 3d7bde19838e..3c0dba072071 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -159,54 +159,47 @@ int qedr_query_device(struct ib_device *ibdev, return 0; } -#define QEDR_SPEED_SDR (1) -#define QEDR_SPEED_DDR (2) -#define QEDR_SPEED_QDR (4) -#define QEDR_SPEED_FDR10 (8) -#define QEDR_SPEED_FDR (16) -#define QEDR_SPEED_EDR (32) - static inline void get_link_speed_and_width(int speed, u8 *ib_speed, u8 *ib_width) { switch (speed) { case 1000: - *ib_speed = QEDR_SPEED_SDR; + *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; break; case 10000: - *ib_speed = QEDR_SPEED_QDR; + *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_1X; break; case 20000: - *ib_speed = QEDR_SPEED_DDR; + *ib_speed = IB_SPEED_DDR; *ib_width = IB_WIDTH_4X; break; case 25000: - *ib_speed = QEDR_SPEED_EDR; + *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_1X; break; case 40000: - *ib_speed = QEDR_SPEED_QDR; + *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_4X; break; case 50000: - *ib_speed = QEDR_SPEED_QDR; - *ib_width = IB_WIDTH_4X; + *ib_speed = IB_SPEED_HDR; + *ib_width = IB_WIDTH_1X; break; case 100000: - *ib_speed = QEDR_SPEED_EDR; + *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_4X; break; default: /* Unsupported */ - *ib_speed = QEDR_SPEED_SDR; + *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; } } -- cgit v1.2.3 From d2183c6f1958e6b6dfdde279f4cee04280710e34 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 20 May 2019 09:05:25 +0300 Subject: RDMA/umem: Move page_shift from ib_umem to ib_odp_umem This value has always been set to PAGE_SHIFT in the core code, the only thing that does differently was the ODP path. Move the value into the ODP struct and still use it for ODP, but change all the non-ODP things to just use PAGE_SHIFT/PAGE_SIZE/PAGE_MASK directly. Reviewed-by: Shiraz Saleem Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_cq.c | 3 +-- drivers/infiniband/hw/hns/hns_roce_srq.c | 10 ++++------ drivers/infiniband/hw/mlx4/mr.c | 8 ++++---- drivers/infiniband/hw/mlx4/srq.c | 2 +- drivers/infiniband/hw/mlx5/mem.c | 20 ++++++++++---------- drivers/infiniband/hw/mlx5/mr.c | 5 +++-- drivers/infiniband/hw/mlx5/odp.c | 23 ++++++++++------------- drivers/infiniband/hw/nes/nes_verbs.c | 9 +++++---- 8 files changed, 38 insertions(+), 42 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 9caf35061721..6e81ff3f1813 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -235,8 +235,7 @@ static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, &buf->hr_mtt); } else { ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem), - (*umem)->page_shift, - &buf->hr_mtt); + PAGE_SHIFT, &buf->hr_mtt); } if (ret) goto err_buf; diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index b3421b1f21e0..ad15b41da30a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -264,8 +264,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, } else ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(srq->umem), - srq->umem->page_shift, - &srq->mtt); + PAGE_SHIFT, &srq->mtt); if (ret) goto err_buf; @@ -291,10 +290,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, ret = hns_roce_mtt_init(hr_dev, npages, page_shift, &srq->idx_que.mtt); } else { - ret = hns_roce_mtt_init(hr_dev, - ib_umem_page_count(srq->idx_que.umem), - srq->idx_que.umem->page_shift, - &srq->idx_que.mtt); + ret = hns_roce_mtt_init( + hr_dev, ib_umem_page_count(srq->idx_que.umem), + PAGE_SHIFT, &srq->idx_que.mtt); } if (ret) { diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 355205a28544..b0b94dedb848 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -258,7 +258,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, int *num_of_mtts) { u64 block_shift = MLX4_MAX_MTT_SHIFT; - u64 min_shift = umem->page_shift; + u64 min_shift = PAGE_SHIFT; u64 last_block_aligned_end = 0; u64 current_block_start = 0; u64 first_block_start = 0; @@ -295,8 +295,8 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, * in access to the wrong data. */ misalignment_bits = - (start_va & (~(((u64)(BIT(umem->page_shift))) - 1ULL))) - ^ current_block_start; + (start_va & (~(((u64)(PAGE_SIZE)) - 1ULL))) ^ + current_block_start; block_shift = min(alignment_of(misalignment_bits), block_shift); } @@ -514,7 +514,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, goto release_mpt_entry; } n = ib_umem_page_count(mmr->umem); - shift = mmr->umem->page_shift; + shift = PAGE_SHIFT; err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr, virt_addr, length, n, shift, diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index 4bf2946b9759..c9f555e04c9f 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -115,7 +115,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, return PTR_ERR(srq->umem); err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), - srq->umem->page_shift, &srq->mtt); + PAGE_SHIFT, &srq->mtt); if (err) goto err_buf; diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 9f90be296ee0..fe1a76d8531c 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -55,9 +55,10 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int i = 0; struct scatterlist *sg; int entry; - unsigned long page_shift = umem->page_shift; if (umem->is_odp) { + unsigned int page_shift = to_ib_umem_odp(umem)->page_shift; + *ncont = ib_umem_page_count(umem); *count = *ncont << (page_shift - PAGE_SHIFT); *shift = page_shift; @@ -67,15 +68,15 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, return; } - addr = addr >> page_shift; + addr = addr >> PAGE_SHIFT; tmp = (unsigned long)addr; m = find_first_bit(&tmp, BITS_PER_LONG); if (max_page_shift) - m = min_t(unsigned long, max_page_shift - page_shift, m); + m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m); for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = sg_dma_len(sg) >> page_shift; - pfn = sg_dma_address(sg) >> page_shift; + len = sg_dma_len(sg) >> PAGE_SHIFT; + pfn = sg_dma_address(sg) >> PAGE_SHIFT; if (base + p != pfn) { /* If either the offset or the new * base are unaligned update m @@ -107,7 +108,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, *ncont = 0; } - *shift = page_shift + m; + *shift = PAGE_SHIFT + m; *count = i; } @@ -140,8 +141,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, int page_shift, size_t offset, size_t num_pages, __be64 *pas, int access_flags) { - unsigned long umem_page_shift = umem->page_shift; - int shift = page_shift - umem_page_shift; + int shift = page_shift - PAGE_SHIFT; int mask = (1 << shift) - 1; int i, k, idx; u64 cur = 0; @@ -165,7 +165,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, i = 0; for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = sg_dma_len(sg) >> umem_page_shift; + len = sg_dma_len(sg) >> PAGE_SHIFT; base = sg_dma_address(sg); /* Skip elements below offset */ @@ -184,7 +184,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, for (; k < len; k++) { if (!(i & mask)) { - cur = base + (k << umem_page_shift); + cur = base + (k << PAGE_SHIFT); cur |= access_flags; idx = (i >> shift) - offset; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 5f09699fab98..4d033796dcfc 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1606,8 +1606,9 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) synchronize_srcu(&dev->mr_srcu); /* Destroy all page mappings */ if (umem_odp->page_list) - mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem), - ib_umem_end(umem)); + mlx5_ib_invalidate_range(umem_odp, + ib_umem_start(umem_odp), + ib_umem_end(umem_odp)); else mlx5_ib_free_implicit_mr(mr); /* diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 91507a2e9290..d0c6f9cc97ef 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -150,7 +150,7 @@ static struct ib_umem_odp *odp_lookup(u64 start, u64 length, if (!rb) goto not_found; odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); - if (ib_umem_start(&odp->umem) > start + length) + if (ib_umem_start(odp) > start + length) goto not_found; } not_found: @@ -200,7 +200,7 @@ void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, static void mr_leaf_free_action(struct work_struct *work) { struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); - int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT; + int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; mr->parent = NULL; @@ -224,7 +224,6 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / sizeof(struct mlx5_mtt)) - 1; u64 idx = 0, blk_start_idx = 0; - struct ib_umem *umem; int in_block = 0; u64 addr; @@ -232,15 +231,14 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, pr_err("invalidation called on NULL umem or non-ODP umem\n"); return; } - umem = &umem_odp->umem; mr = umem_odp->private; if (!mr || !mr->ibmr.pd) return; - start = max_t(u64, ib_umem_start(umem), start); - end = min_t(u64, ib_umem_end(umem), end); + start = max_t(u64, ib_umem_start(umem_odp), start); + end = min_t(u64, ib_umem_end(umem_odp), end); /* * Iteration one - zap the HW's MTTs. The notifiers_count ensures that @@ -249,8 +247,8 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, * but they will write 0s as well, so no difference in the end result. */ - for (addr = start; addr < end; addr += BIT(umem->page_shift)) { - idx = (addr - ib_umem_start(umem)) >> umem->page_shift; + for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { + idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; /* * Strive to write the MTTs in chunks, but avoid overwriting * non-existing MTTs. The huristic here can be improved to @@ -544,13 +542,12 @@ static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end, void *cookie) { struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie; - struct ib_umem *umem = &umem_odp->umem; if (mr->parent != imr) return 0; - ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem), - ib_umem_end(umem)); + ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), + ib_umem_end(umem_odp)); if (umem_odp->dying) return 0; @@ -602,9 +599,9 @@ static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, } next_mr: - size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt); + size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt); - page_shift = mr->umem->page_shift; + page_shift = odp->page_shift; page_mask = ~(BIT(page_shift) - 1); start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; access_mask = ODP_READ_ALLOWED_BIT; diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 49024326a518..ad2b8322cc3f 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -2112,10 +2112,11 @@ static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, return (struct ib_mr *)region; } - nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u," - " offset = %u, page size = %lu.\n", - (unsigned long int)start, (unsigned long int)virt, (u32)length, - ib_umem_offset(region), BIT(region->page_shift)); + nes_debug( + NES_DBG_MR, + "User base = 0x%lX, Virt base = 0x%lX, length = %u, offset = %u, page size = %lu.\n", + (unsigned long)start, (unsigned long)virt, (u32)length, + ib_umem_offset(region), PAGE_SIZE); skip_pages = ((u32)ib_umem_offset(region)) >> 12; -- cgit v1.2.3 From 8ce0048f76542a1f4d68b647c5d4a4d35e7547cf Mon Sep 17 00:00:00 2001 From: Yuval Shaia Date: Sun, 19 May 2019 18:31:27 +0300 Subject: IB/mlx4: Delete unused func arg The function argument virt_addr is not in use - delete it. Signed-off-by: Yuval Shaia Reviewed-by: Majd Dibbiny Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx4/mr.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index b0b94dedb848..753479285ce9 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -368,8 +368,7 @@ end: } static struct ib_umem *mlx4_get_umem_mr(struct ib_udata *udata, u64 start, - u64 length, u64 virt_addr, - int access_flags) + u64 length, int access_flags) { /* * Force registering the memory as writable if the underlying pages @@ -415,8 +414,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = - mlx4_get_umem_mr(udata, start, length, virt_addr, access_flags); + mr->umem = mlx4_get_umem_mr(udata, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err_free; @@ -505,7 +503,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr); ib_umem_release(mmr->umem); - mmr->umem = mlx4_get_umem_mr(udata, start, length, virt_addr, + mmr->umem = mlx4_get_umem_mr(udata, start, length, mr_access_flags); if (IS_ERR(mmr->umem)) { err = PTR_ERR(mmr->umem); -- cgit v1.2.3 From 269c97fd485439702048676326286588c33fd3ba Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:23 +0300 Subject: RDMA/nes: Remove useless NULL checks The destroy functions are always called with relevant structs, there is no need to check their existence. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/nes/nes_verbs.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index ad2b8322cc3f..fb2d0762c7c8 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -1646,9 +1646,6 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) u32 opcode = 0; int ret; - if (ib_cq == NULL) - return 0; - nescq = to_nescq(ib_cq); nesvnic = to_nesvnic(ib_cq->device); nesdev = nesvnic->nesdev; @@ -3708,9 +3705,6 @@ void nes_port_ibevent(struct nes_vnic *nesvnic) */ void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) { - if (nesibdev == NULL) - return; - nes_unregister_ofa_device(nesibdev); ib_dealloc_device(&nesibdev->ibdev); -- cgit v1.2.3 From 3bb58cfe07e273903dde62592ccfb21f9b0761e2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:24 +0300 Subject: RDMA/i40iw: Remove useless NULL checks There is no need to check existence of structures to be destroyed. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 5689d742bafb..a10a30d44b32 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -1070,11 +1070,6 @@ static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct i40iw_device *iwdev; struct i40iw_sc_cq *cq; - if (!ib_cq) { - i40iw_pr_err("ib_cq == NULL\n"); - return 0; - } - iwcq = to_iwcq(ib_cq); iwdev = to_iwdev(ib_cq->device); cq = &iwcq->sc_cq; @@ -2771,9 +2766,6 @@ void i40iw_port_ibevent(struct i40iw_device *iwdev) */ void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev) { - if (!iwibdev) - return; - ib_unregister_device(&iwibdev->ibdev); wait_event_timeout(iwibdev->iwdev->close_wq, !atomic64_read(&iwibdev->iwdev->use_count), -- cgit v1.2.3 From dab99af99c7ca4c9dbea7d4d218ef7eb6b183aaf Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:25 +0300 Subject: RDMA/nes: Remove second wait queue initialization call The same wait queue is initialized a couple of lines above. Fixes: 3c2d774cad5b ("RDMA/nes: Add a driver for NetEffect RNICs") Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/nes/nes_utils.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c index 21b4a8373acf..90f28890246d 100644 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ b/drivers/infiniband/hw/nes/nes_utils.c @@ -586,7 +586,6 @@ struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev) cqp_request->waiting = 0; cqp_request->request_done = 0; cqp_request->callback = 0; - init_waitqueue_head(&cqp_request->waitq); nes_debug(NES_DBG_CQP, "Got cqp request %p from the available list \n", cqp_request); } else -- cgit v1.2.3 From 62a38e704d5720e2b73d0e1d6dfc54441ee75985 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:22 +0300 Subject: RDMA/efa: Remove check that prevents destroy of resources in error flows Drivers cannot check the udata for validity when doing destroy as there will be no way to report this error back to the uverbs. Since udata is new for destroy no driver should start to use it - instead drivers should opt for the ioctl interface and define it in a way where it cannot fail due to incorrect data. Remove the checks on udata construction so EFA is consistent with everything else. Signed-off-by: Leon Romanovsky Acked-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 24 ------------------------ 1 file changed, 24 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 6d6886c9009f..4999a74cee24 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -436,12 +436,6 @@ void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct efa_dev *dev = to_edev(ibpd->device); struct efa_pd *pd = to_epd(ibpd); - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return; - } - ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn); efa_pd_dealloc(dev, pd->pdn); } @@ -459,12 +453,6 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) struct efa_qp *qp = to_eqp(ibqp); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } - ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num); err = efa_destroy_qp_handle(dev, qp->qp_handle); if (err) @@ -865,12 +853,6 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct efa_cq *cq = to_ecq(ibcq); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } - ibdev_dbg(&dev->ibdev, "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n", cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr); @@ -1556,12 +1538,6 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct efa_mr *mr = to_emr(ibmr); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } - ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey); if (mr->umem) { -- cgit v1.2.3 From d34d37d5a102a016c63f904d1ff0284637759cc2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:28 +0300 Subject: RDMA/cxgb3: Use sizeof() notation instead of plain sizeof sizeof(a), sizeof a and sizeof (a) are all valid notations, but first is more readable format recommended by checkpatch.pl. Let's canonize it in cxgb3 drivers, so latter patches won't emit checkpatch warnings. As part of this change, a redundant memset() was removed. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 4 ++-- drivers/infiniband/hw/cxgb3/iwch_cm.c | 2 +- drivers/infiniband/hw/cxgb3/iwch_provider.c | 24 +++++++++++------------- 3 files changed, 14 insertions(+), 16 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 8ac72ac7cbac..e9a5f45dfa14 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -219,7 +219,7 @@ static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) if (!qpid) goto out; for (i = qpid+1; i & rdev_p->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) break; entry->qpid = i; @@ -237,7 +237,7 @@ static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid, { struct cxio_qpid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("%s qpid 0x%x\n", __func__, qpid); diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index 1c90c86fc8b8..0bca72cb4d9a 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -170,7 +170,7 @@ static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb) { struct cpl_tid_release *req; - skb = get_skb(skb, sizeof *req, GFP_KERNEL); + skb = get_skb(skb, sizeof(*req), GFP_KERNEL); if (!skb) return; req = skb_put(skb, sizeof(*req)); diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 3a481dfb1607..82669c254b3f 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -127,7 +127,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, if (udata) { if (!t3a_device(rhp)) { - if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) { + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { kfree(chp); return ERR_PTR(-EFAULT); } @@ -172,7 +172,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, struct iwch_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct iwch_ucontext, ibucontext); - mm = kmalloc(sizeof *mm, GFP_KERNEL); + mm = kmalloc(sizeof(*mm), GFP_KERNEL); if (!mm) { iwch_destroy_cq(&chp->ibcq, udata); return ERR_PTR(-ENOMEM); @@ -185,7 +185,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, spin_unlock(&ucontext->mmap_lock); mm->key = uresp.key; mm->addr = virt_to_phys(chp->cq.queue); - if (udata->outlen < sizeof uresp) { + if (udata->outlen < sizeof(uresp)) { if (!warned++) pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n"); mm->len = PAGE_ALIGN((1UL << uresp.size_log2) * @@ -196,7 +196,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, sizeof(struct t3_cqe)); uresp.memsize = mm->len; uresp.reserved = 0; - resplen = sizeof uresp; + resplen = sizeof(uresp); } if (ib_copy_to_udata(udata, &uresp, resplen)) { kfree(mm); @@ -553,7 +553,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) { pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter)); - if (i == PAGE_SIZE / sizeof *pages) { + if (i == PAGE_SIZE / sizeof(*pages)) { err = iwch_write_pbl(mhp, pages, i, n); if (err) goto pbl_done; @@ -587,7 +587,7 @@ pbl_done: pr_debug("%s user resp pbl_addr 0x%x\n", __func__, uresp.pbl_addr); - if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { iwch_dereg_mr(&mhp->ibmr, udata); err = -EFAULT; goto err; @@ -880,13 +880,13 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, struct iwch_mm_entry *mm1, *mm2; - mm1 = kmalloc(sizeof *mm1, GFP_KERNEL); + mm1 = kmalloc(sizeof(*mm1), GFP_KERNEL); if (!mm1) { iwch_destroy_qp(&qhp->ibqp, udata); return ERR_PTR(-ENOMEM); } - mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); + mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL); if (!mm2) { kfree(mm1); iwch_destroy_qp(&qhp->ibqp, udata); @@ -903,7 +903,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, uresp.db_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); - if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { kfree(mm1); kfree(mm2); iwch_destroy_qp(&qhp->ibqp, udata); @@ -911,7 +911,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, } mm1->key = uresp.key; mm1->addr = virt_to_phys(qhp->wq.queue); - mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr)); + mm1->len = PAGE_ALIGN(wqsize * sizeof(union t3_wr)); insert_mmap(ucontext, mm1); mm2->key = uresp.db_key; mm2->addr = qhp->wq.udb & PAGE_MASK; @@ -932,7 +932,7 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct iwch_dev *rhp; struct iwch_qp *qhp; enum iwch_qp_attr_mask mask = 0; - struct iwch_qp_attributes attrs; + struct iwch_qp_attributes attrs = {}; pr_debug("%s ib_qp %p\n", __func__, ibqp); @@ -944,7 +944,6 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!attr_mask) return 0; - memset(&attrs, 0, sizeof attrs); qhp = to_iwch_qp(ibqp); rhp = qhp->rhp; @@ -1040,7 +1039,6 @@ static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return -EINVAL; dev = to_iwch_dev(ibdev); - memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6); props->hw_ver = dev->rdev.t3cdev_p->type; props->fw_ver = fw_vers_string_to_u64(dev); -- cgit v1.2.3 From 0ddf8f6267ecefc07dace11be8a9d258061c36e1 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:29 +0300 Subject: RDMA/cxgb3: Don't expose DMA addresses DMA addresses like all other kernel addresses should be printed with special %p* formatter. It is needed to allow control of exposure of such information through a dedicated knob. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 6 +++--- drivers/infiniband/hw/cxgb3/iwch_provider.c | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index e9a5f45dfa14..73bc6b8f2a0c 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -565,9 +565,9 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) wqe->sge_cmd = cpu_to_be64(sge_cmd); wqe->ctx1 = cpu_to_be64(ctx1); wqe->ctx0 = cpu_to_be64(ctx0); - pr_debug("CtrlQP dma_addr 0x%llx workq %p size %d\n", - (unsigned long long)rdev_p->ctrl_qp.dma_addr, - rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2); + pr_debug("CtrlQP dma_addr %pad workq %p size %d\n", + &rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq, + 1 << T3_CTRL_QP_SIZE_LOG2); skb->priority = CPL_PRIORITY_CONTROL; return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); err: diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 82669c254b3f..7e6adf01d1d6 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -205,9 +205,9 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, } insert_mmap(ucontext, mm); } - pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n", + pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr %pad\n", chp->cq.cqid, chp, (1 << chp->cq.size_log2), - (unsigned long long)chp->cq.dma_addr); + &chp->cq.dma_addr); return &chp->ibcq; } @@ -919,10 +919,11 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, insert_mmap(ucontext, mm2); } qhp->ibqp.qp_num = qhp->wq.qpid; - pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n", - __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, - qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr, - 1 << qhp->wq.size_log2, qhp->wq.rq_addr); + pr_debug( + "%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr %pad size %d rq_addr 0x%x\n", + __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, + qhp->wq.qpid, qhp, &qhp->wq.dma_addr, 1 << qhp->wq.size_log2, + qhp->wq.rq_addr); return &qhp->ibqp; } -- cgit v1.2.3 From a80287c8130552a837eb61e1ba14844bf2d03156 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:30 +0300 Subject: RDMA/cxgb3: Delete and properly mark unimplemented resize CQ function Resize CQ implementation was guarded by undeclared "notyet" define while cxgb3 was added to the kernel. Twelve years later, this call is still unimplemented, so safely delete it and fix improper return error code when .resize_cq() is not implemented. Fixes: b038ced7b370 ("RDMA/cxgb3: Add driver for Chelsio T3 RNIC") Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 14 ------ drivers/infiniband/hw/cxgb3/cxio_hal.h | 1 - drivers/infiniband/hw/cxgb3/iwch_provider.c | 68 ----------------------------- 3 files changed, 83 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 73bc6b8f2a0c..d9c741fea0e9 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -187,20 +187,6 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel) return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup)); } -#ifdef notyet -int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) -{ - struct rdma_cq_setup setup; - setup.id = cq->cqid; - setup.base_addr = (u64) (cq->dma_addr); - setup.size = 1UL << cq->size_log2; - setup.credits = setup.size; - setup.credit_thres = setup.size; /* TBD: overflow recovery */ - setup.ovfl_mode = 1; - return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup)); -} -#endif - static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) { struct cxio_qpid_list *entry; diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h index c64e50b5a548..5fc26a4648d3 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.h +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h @@ -159,7 +159,6 @@ int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq, enum t3_cq_opcode op, u32 credit); int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq, int kernel); int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq); -int cxio_resize_cq(struct cxio_rdev *rdev, struct t3_cq *cq); void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq, diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 7e6adf01d1d6..4bfab739ec0d 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -211,73 +211,6 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, return &chp->ibcq; } -static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) -{ -#ifdef notyet - struct iwch_cq *chp = to_iwch_cq(cq); - struct t3_cq oldcq, newcq; - int ret; - - pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe); - - /* We don't downsize... */ - if (cqe <= cq->cqe) - return 0; - - /* create new t3_cq with new size */ - cqe = roundup_pow_of_two(cqe+1); - newcq.size_log2 = ilog2(cqe); - - /* Dont allow resize to less than the current wce count */ - if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) { - return -ENOMEM; - } - - /* Quiesce all QPs using this CQ */ - ret = iwch_quiesce_qps(chp); - if (ret) { - return ret; - } - - ret = cxio_create_cq(&chp->rhp->rdev, &newcq); - if (ret) { - return ret; - } - - /* copy CQEs */ - memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) * - sizeof(struct t3_cqe)); - - /* old iwch_qp gets new t3_cq but keeps old cqid */ - oldcq = chp->cq; - chp->cq = newcq; - chp->cq.cqid = oldcq.cqid; - - /* resize new t3_cq to update the HW context */ - ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq); - if (ret) { - chp->cq = oldcq; - return ret; - } - chp->ibcq.cqe = (1<cq.size_log2) - 1; - - /* destroy old t3_cq */ - oldcq.cqid = newcq.cqid; - ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq); - if (ret) { - pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret); - } - - /* add user hooks here */ - - /* resume qps */ - ret = iwch_resume_qps(chp); - return ret; -#else - return -ENOSYS; -#endif -} - static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct iwch_dev *rhp; @@ -1340,7 +1273,6 @@ static const struct ib_device_ops iwch_dev_ops = { .query_port = iwch_query_port, .reg_user_mr = iwch_reg_user_mr, .req_notify_cq = iwch_arm_cq, - .resize_cq = iwch_resize_cq, INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext), }; -- cgit v1.2.3 From 34d568930b87e0dd32c5dbe83a835959f2fc6107 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:31 +0300 Subject: RDMA/cxgb4: Use sizeof() notation Convert various sizeof call sites to be written in standard format sizeof(). Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb4/cm.c | 21 +++++++++------- drivers/infiniband/hw/cxgb4/cq.c | 8 +++--- drivers/infiniband/hw/cxgb4/device.c | 9 +++---- drivers/infiniband/hw/cxgb4/mem.c | 5 ++-- drivers/infiniband/hw/cxgb4/provider.c | 1 - drivers/infiniband/hw/cxgb4/qp.c | 46 ++++++++++++++++------------------ drivers/infiniband/hw/cxgb4/resource.c | 16 ++++++------ 7 files changed, 53 insertions(+), 53 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 0f3b1193d5f8..0147c407ac6c 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -953,7 +953,7 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, mpalen = sizeof(*mpa) + ep->plen; if (mpa_rev_to_use == 2) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(skb, wrlen, GFP_KERNEL); if (!skb) { connect_reply_upcall(ep, -ENOMEM); @@ -997,8 +997,9 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, } if (mpa_rev_to_use == 2) { - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); pr_debug("initiator ird %u ord %u\n", ep->ird, ep->ord); mpa_v2_params.ird = htons((u16)ep->ird); @@ -1057,7 +1058,7 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(NULL, wrlen, GFP_KERNEL); if (!skb) { @@ -1088,8 +1089,9 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons(((u16)ep->ird) | (peer2peer ? MPA_V2_PEER2PEER_MODEL : 0)); @@ -1136,7 +1138,7 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(NULL, wrlen, GFP_KERNEL); if (!skb) { @@ -1171,8 +1173,9 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons((u16)ep->ird); mpa_v2_params.ord = htons((u16)ep->ord); if (peer2peer && (ep->mpa_attr.p2p_type != diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 52ce586621c6..2c0587d8630f 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -43,7 +43,7 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, int wr_len; int ret; - wr_len = sizeof *res_wr + sizeof *res; + wr_len = sizeof(*res_wr) + sizeof(*res); set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = __skb_put_zero(skb, wr_len); @@ -117,7 +117,7 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, } /* build fw_ri_res_wr */ - wr_len = sizeof *res_wr + sizeof *res; + wr_len = sizeof(*res_wr) + sizeof(*res); skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { @@ -1095,10 +1095,10 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, if (ucontext) { ret = -ENOMEM; - mm = kmalloc(sizeof *mm, GFP_KERNEL); + mm = kmalloc(sizeof(*mm), GFP_KERNEL); if (!mm) goto err_remove_handle; - mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); + mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL); if (!mm2) goto err_free_mm; diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 4c0d925c5ff5..a8b9548bd1a2 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -327,7 +327,7 @@ static int qp_open(struct inode *inode, struct file *file) unsigned long index; int count = 1; - qpd = kmalloc(sizeof *qpd, GFP_KERNEL); + qpd = kmalloc(sizeof(*qpd), GFP_KERNEL); if (!qpd) return -ENOMEM; @@ -421,7 +421,7 @@ static int stag_open(struct inode *inode, struct file *file) int ret = 0; int count = 1; - stagd = kmalloc(sizeof *stagd, GFP_KERNEL); + stagd = kmalloc(sizeof(*stagd), GFP_KERNEL); if (!stagd) { ret = -ENOMEM; goto out; @@ -1075,7 +1075,7 @@ static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", DRV_VERSION); - ctx = kzalloc(sizeof *ctx, GFP_KERNEL); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) { ctx = ERR_PTR(-ENOMEM); goto out; @@ -1243,10 +1243,9 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) case CXGB4_STATE_START_RECOVERY: pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev)); if (ctx->dev) { - struct ib_event event; + struct ib_event event = {}; ctx->dev->rdev.flags |= T4_FATAL_ERROR; - memset(&event, 0, sizeof event); event.event = IB_EVENT_DEVICE_FATAL; event.device = &ctx->dev->ibdev; ib_dispatch_event(&event); diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 811c0c8c5b16..fe3733c4026d 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -130,8 +130,9 @@ static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE : len; - wr_len = roundup(sizeof *req + sizeof *sc + - roundup(copy_len, T4_ULPTX_MIN_IO), 16); + wr_len = roundup(sizeof(*req) + sizeof(*sc) + + roundup(copy_len, T4_ULPTX_MIN_IO), + 16); if (!skb) { skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 74b795642fca..8ed75b141521 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -271,7 +271,6 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return -EINVAL; dev = to_c4iw_dev(ibdev); - memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type); props->fw_ver = dev->rdev.lldi.fw_vers; diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index e92b9544357a..41b2e70cd691 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -303,7 +303,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, wq->rq.msn = 1; /* build fw_ri_res_wr */ - wr_len = sizeof *res_wr + 2 * sizeof *res; + wr_len = sizeof(*res_wr) + 2 * sizeof(*res); if (need_rq) wr_len += sizeof(*res); skb = alloc_skb(wr_len, GFP_KERNEL); @@ -439,7 +439,7 @@ static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, rem -= len; } } - len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp); + len = roundup(plen + sizeof(*immdp), 16) - (plen + sizeof(*immdp)); if (len) memset(dstp, 0, len); immdp->op = FW_RI_DATA_IMMD; @@ -528,7 +528,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, T4_MAX_SEND_INLINE, &plen); if (ret) return ret; - size = sizeof wqe->send + sizeof(struct fw_ri_immd) + + size = sizeof(wqe->send) + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, @@ -537,7 +537,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; - size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + + size = sizeof(wqe->send) + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { @@ -545,7 +545,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, wqe->send.u.immd_src[0].r1 = 0; wqe->send.u.immd_src[0].r2 = 0; wqe->send.u.immd_src[0].immdlen = 0; - size = sizeof wqe->send + sizeof(struct fw_ri_immd); + size = sizeof(wqe->send) + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); @@ -579,7 +579,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, T4_MAX_WRITE_INLINE, &plen); if (ret) return ret; - size = sizeof wqe->write + sizeof(struct fw_ri_immd) + + size = sizeof(wqe->write) + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, @@ -588,7 +588,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; - size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + + size = sizeof(wqe->write) + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { @@ -596,7 +596,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, wqe->write.u.immd_src[0].r1 = 0; wqe->write.u.immd_src[0].r2 = 0; wqe->write.u.immd_src[0].immdlen = 0; - size = sizeof wqe->write + sizeof(struct fw_ri_immd); + size = sizeof(wqe->write) + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); @@ -683,7 +683,7 @@ static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr, } wqe->read.r2 = 0; wqe->read.r5 = 0; - *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); + *len16 = DIV_ROUND_UP(sizeof(wqe->read), 16); return 0; } @@ -766,8 +766,8 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); if (ret) return ret; - *len16 = DIV_ROUND_UP(sizeof wqe->recv + - wr->num_sge * sizeof(struct fw_ri_sge), 16); + *len16 = DIV_ROUND_UP( + sizeof(wqe->recv) + wr->num_sge * sizeof(struct fw_ri_sge), 16); return 0; } @@ -886,7 +886,7 @@ static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr, { wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe->inv.r2 = 0; - *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); + *len16 = DIV_ROUND_UP(sizeof(wqe->inv), 16); return 0; } @@ -1606,7 +1606,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe, FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16))); wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; - wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term); + wqe->u.terminate.immdlen = cpu_to_be32(sizeof(*term)); term = (struct terminate_message *)wqe->u.terminate.termmsg; if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) { term->layer_etype = qhp->attr.layer_etype; @@ -1751,16 +1751,15 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) { pr_debug("p2p_type = %d\n", p2p_type); - memset(&init->u, 0, sizeof init->u); + memset(&init->u, 0, sizeof(init->u)); switch (p2p_type) { case FW_RI_INIT_P2PTYPE_RDMA_WRITE: init->u.write.opcode = FW_RI_RDMA_WRITE_WR; init->u.write.stag_sink = cpu_to_be32(1); init->u.write.to_sink = cpu_to_be64(1); init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD; - init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write + - sizeof(struct fw_ri_immd), - 16); + init->u.write.len16 = DIV_ROUND_UP( + sizeof(init->u.write) + sizeof(struct fw_ri_immd), 16); break; case FW_RI_INIT_P2PTYPE_READ_REQ: init->u.write.opcode = FW_RI_RDMA_READ_WR; @@ -1768,7 +1767,7 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) init->u.read.to_src_lo = cpu_to_be32(1); init->u.read.stag_sink = cpu_to_be32(1); init->u.read.to_sink_lo = cpu_to_be32(1); - init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16); + init->u.read.len16 = DIV_ROUND_UP(sizeof(init->u.read), 16); break; } } @@ -1782,7 +1781,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp, qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord); - skb = alloc_skb(sizeof *wqe, GFP_KERNEL); + skb = alloc_skb(sizeof(*wqe), GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto out; @@ -2302,7 +2301,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ucontext->key += PAGE_SIZE; } spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); + ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (ret) goto err_free_ma_sync_key; sq_key_mm->key = uresp.sq_key; @@ -2386,7 +2385,7 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct c4iw_dev *rhp; struct c4iw_qp *qhp; enum c4iw_qp_attr_mask mask = 0; - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {}; pr_debug("ib_qp %p\n", ibqp); @@ -2398,7 +2397,6 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!attr_mask) return 0; - memset(&attrs, 0, sizeof attrs); qhp = to_c4iw_qp(ibqp); rhp = qhp->rhp; @@ -2482,8 +2480,8 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, { struct c4iw_qp *qhp = to_c4iw_qp(ibqp); - memset(attr, 0, sizeof *attr); - memset(init_attr, 0, sizeof *init_attr); + memset(attr, 0, sizeof(*attr)); + memset(init_attr, 0, sizeof(*init_attr)); attr->qp_state = to_ib_qp_state(qhp->attr.state); init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c index 57ed26b3cc21..5c95c789f302 100644 --- a/drivers/infiniband/hw/cxgb4/resource.c +++ b/drivers/infiniband/hw/cxgb4/resource.c @@ -126,7 +126,7 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) rdev->stats.qid.cur += rdev->qpmask + 1; mutex_unlock(&rdev->stats.lock); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -137,13 +137,13 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) * now put the same ids on the qp list since they all * map to the same db/gts page. */ - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->qpids); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -165,7 +165,7 @@ void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid, { struct c4iw_qid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("qid 0x%x\n", qid); @@ -200,7 +200,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) rdev->stats.qid.cur += rdev->qpmask + 1; mutex_unlock(&rdev->stats.lock); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -211,13 +211,13 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) * now put the same ids on the cq list since they all * map to the same db/gts page. */ - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->cqids); for (i = qid; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -239,7 +239,7 @@ void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid, { struct c4iw_qid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("qid 0x%x\n", qid); -- cgit v1.2.3 From cae626b97851afc2219e7607183a9a23cbba3bef Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 20 May 2019 09:54:32 +0300 Subject: RDMA/cxgb4: Don't expose DMA addresses Change unconditional print of DMA address to be printed with special printk format type specifier. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb4/cq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 2c0587d8630f..6557e7c5af66 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1135,9 +1135,9 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, mm2->len = PAGE_SIZE; insert_mmap(ucontext, mm2); } - pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n", - chp->cq.cqid, chp, chp->cq.size, - chp->cq.memsize, (unsigned long long)chp->cq.dma_addr); + pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr %pad\n", + chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize, + &chp->cq.dma_addr); return &chp->ibcq; err_free_mm2: kfree(mm2); -- cgit v1.2.3 From f70baa7ee3d1b5a9e66ac7549e31641a656f23c1 Mon Sep 17 00:00:00 2001 From: Nirranjan Kirubaharan Date: Thu, 23 May 2019 00:05:39 -0700 Subject: iw_cxgb4: Fix qpid leak Add await in destroy_qp() so that all references to qp are dereferenced and qp is freed in destroy_qp() itself. This ensures freeing of all QPs before invocation of dealloc_ucontext(), which prevents loss of in use qpids stored in the ucontext. Signed-off-by: Nirranjan Kirubaharan Reviewed-by: Potnuri Bharat Teja Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 4 +-- drivers/infiniband/hw/cxgb4/qp.c | 48 ++++++++++++---------------------- 2 files changed, 19 insertions(+), 33 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 916ef982172e..cf7512b2c4c0 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -490,13 +490,13 @@ struct c4iw_qp { struct t4_wq wq; spinlock_t lock; struct mutex mutex; - struct kref kref; wait_queue_head_t wait; int sq_sig_all; struct c4iw_srq *srq; - struct work_struct free_work; struct c4iw_ucontext *ucontext; struct c4iw_wr_wait *wr_waitp; + struct completion qp_rel_comp; + refcount_t qp_refcnt; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index 41b2e70cd691..9523986d62b1 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -890,43 +890,17 @@ static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr, return 0; } -static void free_qp_work(struct work_struct *work) -{ - struct c4iw_ucontext *ucontext; - struct c4iw_qp *qhp; - struct c4iw_dev *rhp; - - qhp = container_of(work, struct c4iw_qp, free_work); - ucontext = qhp->ucontext; - rhp = qhp->rhp; - - pr_debug("qhp %p ucontext %p\n", qhp, ucontext); - destroy_qp(&rhp->rdev, &qhp->wq, - ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq); - - c4iw_put_wr_wait(qhp->wr_waitp); - kfree(qhp); -} - -static void queue_qp_free(struct kref *kref) -{ - struct c4iw_qp *qhp; - - qhp = container_of(kref, struct c4iw_qp, kref); - pr_debug("qhp %p\n", qhp); - queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); -} - void c4iw_qp_add_ref(struct ib_qp *qp) { pr_debug("ib_qp %p\n", qp); - kref_get(&to_c4iw_qp(qp)->kref); + refcount_inc(&to_c4iw_qp(qp)->qp_refcnt); } void c4iw_qp_rem_ref(struct ib_qp *qp) { pr_debug("ib_qp %p\n", qp); - kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); + if (refcount_dec_and_test(&to_c4iw_qp(qp)->qp_refcnt)) + complete(&to_c4iw_qp(qp)->qp_rel_comp); } static void add_to_fc_list(struct list_head *head, struct list_head *entry) @@ -2098,10 +2072,12 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_qp *qhp; + struct c4iw_ucontext *ucontext; struct c4iw_qp_attributes attrs; qhp = to_c4iw_qp(ib_qp); rhp = qhp->rhp; + ucontext = qhp->ucontext; attrs.next_state = C4IW_QP_STATE_ERROR; if (qhp->attr.state == C4IW_QP_STATE_TERMINATE) @@ -2119,7 +2095,17 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) c4iw_qp_rem_ref(ib_qp); + wait_for_completion(&qhp->qp_rel_comp); + pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp, qhp->wq.sq.qid); + pr_debug("qhp %p ucontext %p\n", qhp, ucontext); + + destroy_qp(&rhp->rdev, &qhp->wq, + ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq); + + c4iw_put_wr_wait(qhp->wr_waitp); + + kfree(qhp); return 0; } @@ -2229,8 +2215,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, spin_lock_init(&qhp->lock); mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); - kref_init(&qhp->kref); - INIT_WORK(&qhp->free_work, free_qp_work); + init_completion(&qhp->qp_rel_comp); + refcount_set(&qhp->qp_refcnt, 1); ret = xa_insert_irq(&rhp->qps, qhp->wq.sq.qid, qhp, GFP_KERNEL); if (ret) -- cgit v1.2.3 From 8ffb813255c422f4d05be227ab4443cb6054a078 Mon Sep 17 00:00:00 2001 From: Yixian Liu Date: Fri, 24 May 2019 15:31:20 +0800 Subject: RDMA/hns: Remove unnecessary print message in aeq There is no need to print when communication is established, especially while lots of qp used by application. Signed-off-by: Yixian Liu Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index b5392cb5b20f..1c7e6e6f6972 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4682,7 +4682,6 @@ static void hns_roce_irq_work_handle(struct work_struct *work) dev_warn(dev, "Path migration failed.\n"); break; case HNS_ROCE_EVENT_TYPE_COMM_EST: - dev_info(dev, "Communication established.\n"); break; case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: dev_warn(dev, "Send queue drained.\n"); -- cgit v1.2.3 From 0502849d0bb133b492eed24fd270441e652c84cc Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Fri, 24 May 2019 15:31:21 +0800 Subject: RDMA/hns: Update CQE specifications According to hip08 UM, the maximum number of CQEs supported by each CQ is 4M. Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index edfdbe2ce0db..0ac177ad6b79 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -54,7 +54,7 @@ #define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 #define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100 #define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 -#define HNS_ROCE_V2_MAX_CQE_NUM 0x10000 +#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 #define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000 #define HNS_ROCE_V2_MAX_RQ_SGE_NUM 0x100 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 0xff -- cgit v1.2.3 From 780f33962ef27d7f27c6b47a55593c6ffd357922 Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Fri, 24 May 2019 15:31:22 +0800 Subject: RDMA/hns: Move spin_lock_irqsave to the correct place When hip08 set gid, it will call spin_unlock_bh when send cmq. if main.ko call spin_lock_irqsave firstly, and the kernel is before commit f71b74bca637 ("irq/softirqs: Use lockdep to assert IRQs are disabled/enabled"), it will cause WARN_ON_ONCE because of calling spin_unlock_bh in disable context. In fact, the spin_lock_irqsave in main.ko is only used for hip06, and should be placed in hns_roce_hw_v1.c. hns_roce_hw_v2.c uses its own spin_unlock_bh and does not need main.ko manage spin_lock. Signed-off-by: Lang Cheng Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 5 +++++ drivers/infiniband/hw/hns/hns_roce_main.c | 10 ---------- 2 files changed, 5 insertions(+), 10 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 4c5d0f160c10..c35a4de6d4e6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -1741,11 +1741,14 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr) { + unsigned long flags; u32 *p = NULL; u8 gid_idx = 0; gid_idx = hns_get_gid_index(hr_dev, port, gid_index); + spin_lock_irqsave(&hr_dev->iboe.lock, flags); + p = (u32 *)&gid->raw[0]; roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG + (HNS_ROCE_V1_GID_NUM * gid_idx)); @@ -1762,6 +1765,8 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + (HNS_ROCE_V1_GID_NUM * gid_idx)); + spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); + return 0; } diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 8da5f18bf820..05a0f7d50ce3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -78,18 +78,13 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) { struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); u8 port = attr->port_num - 1; - unsigned long flags; int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; - spin_lock_irqsave(&hr_dev->iboe.lock, flags); - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr); - spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return ret; } @@ -98,18 +93,13 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); struct ib_gid_attr zattr = { }; u8 port = attr->port_num - 1; - unsigned long flags; int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; - spin_lock_irqsave(&hr_dev->iboe.lock, flags); - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr); - spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return ret; } -- cgit v1.2.3 From 669cefb654cb69b280e31380f5fc7e3b5755b0cd Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Fri, 24 May 2019 15:31:23 +0800 Subject: RDMA/hns: Remove jiffies operation in disable interrupt context In some functions, the jiffies operation is unnecessary, and we can control delay using mdelay and udelay functions only. Especially, in hns_roce_v1_clear_hem, the function calls spin_lock_irqsave, the context disables interrupt, so we can not use jiffies and msleep functions. Signed-off-by: Lang Cheng Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hem.c | 21 +++++++++++---------- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 19 ++++++++++--------- 2 files changed, 21 insertions(+), 19 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 8e29dbb5b5fb..d0eacd8c2575 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -376,18 +376,19 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; - while (1) { - if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (!(time_before(jiffies, end))) { - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); - spin_unlock_irqrestore(lock, flags); - return -EBUSY; - } - } else { + end = HW_SYNC_TIMEOUT_MSECS; + while (end) { + if (!readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) break; - } + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; + } + + if (end <= 0) { + dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); + spin_unlock_irqrestore(lock, flags); + return -EBUSY; } bt_cmd_l = (u32)bt_ba; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index c35a4de6d4e6..f13c9c3e56d4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -965,8 +965,7 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) struct hns_roce_free_mr *free_mr; struct hns_roce_v1_priv *priv; struct completion comp; - unsigned long end = - msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies; + unsigned long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS; priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; @@ -986,10 +985,11 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) queue_work(free_mr->free_mr_wq, &(lp_qp_work->work)); - while (time_before_eq(jiffies, end)) { + while (end) { if (try_wait_for_completion(&comp)) return 0; msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE); + end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE; } lp_qp_work->comp_flag = 0; @@ -1103,8 +1103,7 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_free_mr *free_mr; struct hns_roce_v1_priv *priv; struct completion comp; - unsigned long end = - msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies; + unsigned long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS; unsigned long start = jiffies; int npages; int ret = 0; @@ -1134,10 +1133,11 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, queue_work(free_mr->free_mr_wq, &(mr_work->work)); - while (time_before_eq(jiffies, end)) { + while (end) { if (try_wait_for_completion(&comp)) goto free_mr; msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE); + end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE; } mr_work->comp_flag = 0; @@ -2462,10 +2462,10 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; + end = HW_SYNC_TIMEOUT_MSECS; while (1) { if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (!(time_before(jiffies, end))) { + if (end < 0) { dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); @@ -2474,7 +2474,8 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, } else { break; } - msleep(HW_SYNC_SLEEP_TIME_INTERVAL); + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; } bt_cmd_val[0] = (__le32)bt_ba; -- cgit v1.2.3 From 2a3d923f87303b2d49c6cfb7bf6a25144a1ce265 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Fri, 24 May 2019 23:29:36 +0800 Subject: RDMA/hns: Replace magic numbers with #defines This patch makes the code more readable by removing magic numbers. Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_db.c | 8 ++-- drivers/infiniband/hw/hns/hns_roce_device.h | 37 +++++++++++++--- drivers/infiniband/hw/hns/hns_roce_hem.c | 18 ++++---- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 65 ++++++++++++++++------------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 4 ++ drivers/infiniband/hw/hns/hns_roce_main.c | 7 ++-- drivers/infiniband/hw/hns/hns_roce_mr.c | 43 ++++++++++--------- 8 files changed, 115 insertions(+), 69 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index 0c6c1fe87705..3a040a9390d8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -78,7 +78,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( if (!pgdir) return NULL; - bitmap_fill(pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2); + bitmap_fill(pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); pgdir->bits[0] = pgdir->order0; pgdir->bits[1] = pgdir->order1; pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, @@ -116,7 +117,7 @@ found: db->u.pgdir = pgdir; db->index = i; db->db_record = pgdir->page + db->index; - db->dma = pgdir->db_dma + db->index * 4; + db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; db->order = order; return 0; @@ -170,7 +171,8 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) i >>= o; set_bit(i, db->u.pgdir->bits[o]); - if (bitmap_full(db->u.pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2)) { + if (bitmap_full(db->u.pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 563cf39df6d5..d6e8b446fc57 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -37,9 +37,12 @@ #define DRV_NAME "hns_roce" +/* hip08 is a pci device, it includes two version according pci version id */ +#define PCI_REVISION_ID_HIP08_A 0x20 +#define PCI_REVISION_ID_HIP08_B 0x21 + #define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') -#define MAC_ADDR_OCTET_NUM 6 #define HNS_ROCE_MAX_MSG_LEN 0x80000000 #define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b)) @@ -48,6 +51,10 @@ #define HNS_ROCE_BA_SIZE (32 * 4096) +#define BA_BYTE_LEN 8 + +#define BITS_PER_BYTE 8 + /* Hardware specification only for v1 engine */ #define HNS_ROCE_MIN_CQE_NUM 0x40 #define HNS_ROCE_MIN_WQE_NUM 0x20 @@ -55,6 +62,7 @@ /* Hardware specification only for v1 engine */ #define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7 #define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000 +#define HNS_ROCE_MAX_SGE_NUM 2 #define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20 #define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \ @@ -64,6 +72,9 @@ #define HNS_ROCE_MAX_IRQ_NUM 128 +#define HNS_ROCE_SGE_IN_WQE 2 +#define HNS_ROCE_SGE_SHIFT 4 + #define EQ_ENABLE 1 #define EQ_DISABLE 0 @@ -81,6 +92,7 @@ #define HNS_ROCE_MAX_PORTS 6 #define HNS_ROCE_MAX_GID_NUM 16 #define HNS_ROCE_GID_SIZE 16 +#define HNS_ROCE_SGE_SIZE 16 #define HNS_ROCE_HOP_NUM_0 0xff @@ -111,6 +123,8 @@ #define PAGES_SHIFT_24 24 #define PAGES_SHIFT_32 32 +#define HNS_ROCE_PCI_BAR_NUM 2 + #define HNS_ROCE_IDX_QUE_ENTRY_SZ 4 #define SRQ_DB_REG 0x230 @@ -213,6 +227,9 @@ enum hns_roce_mtt_type { MTT_TYPE_IDX }; +#define HNS_ROCE_DB_TYPE_COUNT 2 +#define HNS_ROCE_DB_UNIT_SIZE 4 + enum { HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4 }; @@ -413,8 +430,8 @@ struct hns_roce_buf { struct hns_roce_db_pgdir { struct list_head list; DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE); - DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2); - unsigned long *bits[2]; + DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); + unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT]; u32 *page; dma_addr_t db_dma; }; @@ -535,7 +552,7 @@ struct hns_roce_av { u8 hop_limit; __le32 sl_tclass_flowlabel; u8 dgid[HNS_ROCE_GID_SIZE]; - u8 mac[6]; + u8 mac[ETH_ALEN]; __le16 vlan; bool vlan_en; }; @@ -940,6 +957,16 @@ struct hns_roce_hw { const struct ib_device_ops *hns_roce_dev_srq_ops; }; +enum hns_phy_state { + HNS_ROCE_PHY_SLEEP = 1, + HNS_ROCE_PHY_POLLING = 2, + HNS_ROCE_PHY_DISABLED = 3, + HNS_ROCE_PHY_TRAINING = 4, + HNS_ROCE_PHY_LINKUP = 5, + HNS_ROCE_PHY_LINKERR = 6, + HNS_ROCE_PHY_TEST = 7 +}; + struct hns_roce_dev { struct ib_device ib_dev; struct platform_device *pdev; @@ -962,7 +989,7 @@ struct hns_roce_dev { struct hns_roce_caps caps; struct xarray qp_table_xa; - unsigned char dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM]; + unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN]; u64 sys_image_guid; u32 vendor_id; u32 vendor_part_id; diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index d0eacd8c2575..157c84a1f55f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -165,7 +165,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.mtt_hop_num; break; case HEM_TYPE_CQE: @@ -173,7 +173,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.cqe_hop_num; break; case HEM_TYPE_SRQWQE: @@ -181,7 +181,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.srqwqe_hop_num; break; case HEM_TYPE_IDX: @@ -189,7 +189,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.idx_hop_num; break; default: @@ -206,7 +206,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, * MTT/CQE alloc hem for bt pages. */ bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); - chunk_ba_num = mhop->bt_chunk_size / 8; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : mhop->bt_chunk_size; table_idx = (*obj & (table->num_obj - 1)) / @@ -436,7 +436,7 @@ static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, buf_chunk_size = mhop.buf_chunk_size; bt_chunk_size = mhop.bt_chunk_size; hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / 8; + chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; bt_num = hns_roce_get_bt_num(table->type, hop_num); switch (bt_num) { @@ -646,7 +646,7 @@ static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, bt_chunk_size = mhop.bt_chunk_size; hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / 8; + chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; bt_num = hns_roce_get_bt_num(table->type, hop_num); switch (bt_num) { @@ -800,7 +800,7 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, i = mhop.l0_idx; j = mhop.l1_idx; if (mhop.hop_num == 2) - hem_idx = i * (mhop.bt_chunk_size / 8) + j; + hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; else if (mhop.hop_num == 1 || mhop.hop_num == HNS_ROCE_HOP_NUM_0) hem_idx = i; @@ -1000,7 +1000,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, } obj_per_chunk = buf_chunk_size / obj_size; num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; - bt_chunk_num = bt_chunk_size / 8; + bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; if (type >= HEM_TYPE_MTT) num_bt_l0 = bt_chunk_num; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index f13c9c3e56d4..878c8ae35630 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -818,7 +818,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) attr.dest_qp_num = hr_qp->qpn; memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr), hr_dev->dev_addr[port], - MAC_ADDR_OCTET_NUM); + ETH_ALEN); memcpy(&dgid.raw, &subnet_prefix, sizeof(u64)); memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 1c7e6e6f6972..7fcec998618a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3426,7 +3426,9 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, else roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, - V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ? + V2_QPC_BYTE_4_SGE_SHIFT_S, + hr_qp->sq.max_gs > + HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ? ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, @@ -3708,13 +3710,14 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); - memcpy(&(context->dmac), dmac, 4); + memcpy(&(context->dmac), dmac, sizeof(u32)); roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); qpc_mask->dmac = 0; roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, V2_QPC_BYTE_52_DMAC_S, 0); + /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */ roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, V2_QPC_BYTE_56_LP_PKTN_INI_S, 4); roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, @@ -3756,6 +3759,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); + /* rocee send 2^lp_sgen_ini segs every time */ roce_set_field(context->byte_168_irrl_idx, V2_QPC_BYTE_168_LP_SGEN_INI_M, V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); @@ -3810,14 +3814,15 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - context->sq_cur_sge_blk_addr = - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? - ((u32)(mtts[hr_qp->sge.offset / page_size] - >> PAGE_ADDR_SHIFT)) : 0; + context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) || + hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? + ((u32)(mtts[hr_qp->sge.offset / page_size] >> + PAGE_ADDR_SHIFT)) : 0; roce_set_field(context->byte_184_irrl_idx, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? + ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > + HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? (mtts[hr_qp->sge.offset / page_size] >> (32 + PAGE_ADDR_SHIFT)) : 0); qpc_mask->sq_cur_sge_blk_addr = 0; @@ -4144,7 +4149,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, roce_set_field(context->byte_224_retry_msg, V2_QPC_BYTE_224_RETRY_MSG_PSN_M, V2_QPC_BYTE_224_RETRY_MSG_PSN_S, - attr->sq_psn >> 16); + attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S); roce_set_field(qpc_mask->byte_224_retry_msg, V2_QPC_BYTE_224_RETRY_MSG_PSN_M, V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); @@ -4374,11 +4379,12 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S); qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_RRE_S)) << 2) | - ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_RWE_S)) << 1) | - ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_ATE_S)) << 3); + V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); + if (hr_qp->ibqp.qp_type == IB_QPT_RC || hr_qp->ibqp.qp_type == IB_QPT_UC) { struct ib_global_route *grh = @@ -5150,8 +5156,8 @@ static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * (bt_chk_sz / 8) + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * (bt_chk_sz / BA_BYTE_LEN) + j; if ((i == eq->l0_last_num - 1) && j == eq->l1_last_num - 1) { eqe_alloc = (buf_chk_sz / eq->eqe_size) @@ -5367,9 +5373,9 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); - ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1) - / buf_chk_sz; - bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8); + ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size), + buf_chk_sz); + bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN); /* hop_num = 0 */ if (mhop_num == HNS_ROCE_HOP_NUM_0) { @@ -5414,12 +5420,12 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, goto err_dma_alloc_l0; if (mhop_num == 1) { - if (ba_num > (bt_chk_sz / 8)) + if (ba_num > (bt_chk_sz / BA_BYTE_LEN)) dev_err(dev, "ba_num %d is too large for 1 hop\n", ba_num); /* alloc buf */ - for (i = 0; i < bt_chk_sz / 8; i++) { + for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { if (eq_buf_cnt + 1 < ba_num) { size = buf_chk_sz; } else { @@ -5443,7 +5449,7 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, } else if (mhop_num == 2) { /* alloc L1 BT and buf */ - for (i = 0; i < bt_chk_sz / 8; i++) { + for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz, &(eq->l1_dma[i]), GFP_KERNEL); @@ -5451,8 +5457,8 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, goto err_dma_alloc_l1; *(eq->bt_l0 + i) = eq->l1_dma[i]; - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * bt_chk_sz / 8 + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * bt_chk_sz / BA_BYTE_LEN + j; if (eq_buf_cnt + 1 < ba_num) { size = buf_chk_sz; } else { @@ -5497,8 +5503,8 @@ err_dma_alloc_l1: dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * bt_chk_sz / 8 + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * bt_chk_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], eq->buf_dma[idx]); } @@ -5521,11 +5527,11 @@ err_dma_alloc_buf: dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { if (i == record_i && j >= record_j) break; - idx = i * bt_chk_sz / 8 + j; + idx = i * bt_chk_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], eq->buf_dma[idx]); @@ -5982,7 +5988,7 @@ static int find_empty_entry(struct hns_roce_idx_que *idx_que) bit_num = ffs(idx_que->bitmap[i]); idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1)); - return i * sizeof(u64) * 8 + (bit_num - 1); + return i * BITS_PER_LONG_LONG + (bit_num - 1); } static void fill_idx_queue(struct hns_roce_idx_que *idx_que, @@ -6058,7 +6064,8 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, */ wmb(); - srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << 24 | srq->srqn; + srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | + (srq->srqn & V2_DB_BYTE_4_TAG_M); srq_db.parameter = srq->head; hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index 0ac177ad6b79..bce21fd2ebb6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -886,6 +886,10 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16 #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16) +#define V2_QP_RWE_S 1 /* rdma write enable */ +#define V2_QP_RRE_S 2 /* rdma read enable */ +#define V2_QP_ATE_S 3 /* rdma atomic enable */ + struct hns_roce_v2_cqe { __le32 byte_4; union { diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 05a0f7d50ce3..a6c5c67d0b87 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -64,10 +64,10 @@ static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) u8 phy_port; u32 i = 0; - if (!memcmp(hr_dev->dev_addr[port], addr, MAC_ADDR_OCTET_NUM)) + if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN)) return 0; - for (i = 0; i < MAC_ADDR_OCTET_NUM; i++) + for (i = 0; i < ETH_ALEN; i++) hr_dev->dev_addr[port][i] = addr[i]; phy_port = hr_dev->iboe.phy_port[port]; @@ -262,7 +262,8 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ? IB_PORT_ACTIVE : IB_PORT_DOWN; - props->phys_state = (props->state == IB_PORT_ACTIVE) ? 5 : 3; + props->phys_state = (props->state == IB_PORT_ACTIVE) ? + HNS_ROCE_PHY_LINKUP : HNS_ROCE_PHY_DISABLED; spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 6110ec408626..38ed4ac741b5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -314,11 +314,11 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { if (i == loop_i && j >= loop_j) break; - bt_idx = i * pbl_bt_sz / 8 + j; + bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -329,8 +329,8 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { - bt_idx = i * pbl_bt_sz / 8 + j; + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { + bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -533,7 +533,7 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, { struct device *dev = hr_dev->dev; unsigned long index = 0; - int ret = 0; + int ret; /* Allocate a key for mr from mr_table */ ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); @@ -559,7 +559,8 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, mr->pbl_l0_dma_addr = 0; } else { if (!hr_dev->caps.pbl_hop_num) { - mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, + mr->pbl_buf = dma_alloc_coherent(dev, + npages * BA_BYTE_LEN, &(mr->pbl_dma_addr), GFP_KERNEL); if (!mr->pbl_buf) @@ -590,9 +591,8 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, if (mhop_num == HNS_ROCE_HOP_NUM_0) return; - /* hop_num = 1 */ if (mhop_num == 1) { - dma_free_coherent(dev, (unsigned int)(npages * 8), + dma_free_coherent(dev, (unsigned int)(npages * BA_BYTE_LEN), mr->pbl_buf, mr->pbl_dma_addr); return; } @@ -603,12 +603,13 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, if (mhop_num == 2) { for (i = 0; i < mr->l0_chunk_last_num; i++) { if (i == mr->l0_chunk_last_num - 1) { - npages_allocated = i * (pbl_bt_sz / 8); + npages_allocated = + i * (pbl_bt_sz / BA_BYTE_LEN); dma_free_coherent(dev, - (npages - npages_allocated) * 8, - mr->pbl_bt_l1[i], - mr->pbl_l1_dma_addr[i]); + (npages - npages_allocated) * BA_BYTE_LEN, + mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); break; } @@ -621,16 +622,17 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { - bt_idx = i * (pbl_bt_sz / 8) + j; + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { + bt_idx = i * (pbl_bt_sz / BA_BYTE_LEN) + j; if ((i == mr->l0_chunk_last_num - 1) && j == mr->l1_chunk_last_num - 1) { npages_allocated = bt_idx * - (pbl_bt_sz / 8); + (pbl_bt_sz / BA_BYTE_LEN); dma_free_coherent(dev, - (npages - npages_allocated) * 8, + (npages - npages_allocated) * + BA_BYTE_LEN, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -675,7 +677,8 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, npages = ib_umem_page_count(mr->umem); if (!hr_dev->caps.pbl_hop_num) - dma_free_coherent(dev, (unsigned int)(npages * 8), + dma_free_coherent(dev, + (unsigned int)(npages * BA_BYTE_LEN), mr->pbl_buf, mr->pbl_dma_addr); else hns_roce_mhop_free(hr_dev, mr); @@ -1059,6 +1062,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { page_addr = sg_page_iter_dma_address(&sg_iter); if (!hr_dev->caps.pbl_hop_num) { + /* for hip06, page addr is aligned to 4K */ mr->pbl_buf[i++] = page_addr >> 12; } else if (hr_dev->caps.pbl_hop_num == 1) { mr->pbl_buf[i++] = page_addr; @@ -1069,7 +1073,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, mr->pbl_bt_l2[i][j] = page_addr; j++; - if (j >= (pbl_bt_sz / 8)) { + if (j >= (pbl_bt_sz / BA_BYTE_LEN)) { i++; j = 0; } @@ -1117,7 +1121,8 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, } else { u64 pbl_size = 1; - bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8; + bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / + BA_BYTE_LEN; for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) pbl_size *= bt_size; if (n > pbl_size) { -- cgit v1.2.3 From cfcc048ca76e4927f163f178f59d557588ba32f7 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 25 May 2019 20:57:37 +0800 Subject: IB/hfi1: Remove set but not used variables 'offset' and 'fspsn' Fixes gcc '-Wunused-but-set-variable' warning: drivers/infiniband/hw/hfi1/tid_rdma.c: In function tid_rdma_rcv_error: drivers/infiniband/hw/hfi1/tid_rdma.c:2029:7: warning: variable offset set but not used [-Wunused-but-set-variable] drivers/infiniband/hw/hfi1/tid_rdma.c: In function hfi1_rc_rcv_tid_rdma_ack: drivers/infiniband/hw/hfi1/tid_rdma.c:4555:35: warning: variable fspsn set but not used [-Wunused-but-set-variable] 'offset' is never used since introduction in commit d0d564a1caac ("IB/hfi1: Add functions to receive TID RDMA READ request") 'fspsn' is never used since introduciotn in commit 9e93e967f7b4 ("IB/hfi1: Add a function to receive TID RDMA ACK packet") Signed-off-by: YueHaibing Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/tid_rdma.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index 6fb93032fbef..bdf1c313e13f 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -2026,7 +2026,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet, trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req); if (e->opcode == TID_OP(READ_REQ)) { struct ib_reth *reth; - u32 offset; u32 len; u32 rkey; u64 vaddr; @@ -2038,7 +2037,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet, * The requester always restarts from the start of the original * request. */ - offset = delta_psn(psn, e->psn) * qp->pmtu; len = be32_to_cpu(reth->length); if (psn != e->psn || len != req->total_len) goto unlock; @@ -4552,7 +4550,7 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet) struct rvt_swqe *wqe; struct tid_rdma_request *req; struct tid_rdma_flow *flow; - u32 aeth, psn, req_psn, ack_psn, fspsn, resync_psn, ack_kpsn; + u32 aeth, psn, req_psn, ack_psn, resync_psn, ack_kpsn; unsigned long flags; u16 fidx; @@ -4756,7 +4754,6 @@ done: IB_AETH_CREDIT_MASK) { case 0: /* PSN sequence error */ flow = &req->flows[req->acked_tail]; - fspsn = full_flow_psn(flow, flow->flow_state.spsn); trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); req->r_ack_psn = mask_psn(be32_to_cpu(ohdr->bth[2])); -- cgit v1.2.3 From ea996974589e7987eb463d8a7c404358244755ea Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 24 May 2019 18:45:22 -0700 Subject: RDMA: Convert put_page() to put_user_page*() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For infiniband code that retains pages via get_user_pages*(), release those pages via the new put_user_page(), or put_user_pages*(), instead of put_page() This is a tiny part of the second step of fixing the problem described in [1]. The steps are: 1) Provide put_user_page*() routines, intended to be used for releasing pages that were pinned via get_user_pages*(). 2) Convert all of the call sites for get_user_pages*(), to invoke put_user_page*(), instead of put_page(). This involves dozens of call sites, and will take some time. 3) After (2) is complete, use get_user_pages*() and put_user_page*() to implement tracking of these pages. This tracking will be separate from the existing struct page refcounting. 4) Use the tracking and identification of these pages, to implement special handling (especially in writeback paths) when the pages are backed by a filesystem. Again, [1] provides details as to why that is desirable. [1] https://lwn.net/Articles/753027/ : "The Trouble with get_user_pages()" Reviewed-by: Jan Kara Reviewed-by: Dennis Dalessandro Reviewed-by: Ira Weiny Reviewed-by: Jérôme Glisse Acked-by: Jason Gunthorpe Tested-by: Ira Weiny Signed-off-by: John Hubbard Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/user_pages.c | 11 ++++------- drivers/infiniband/hw/mthca/mthca_memfree.c | 6 +++--- drivers/infiniband/hw/qib/qib_user_pages.c | 11 ++++------- drivers/infiniband/hw/qib/qib_user_sdma.c | 6 +++--- drivers/infiniband/hw/usnic/usnic_uiom.c | 7 ++++--- 5 files changed, 18 insertions(+), 23 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c index 02eee8eff1db..b89a9b9aef7a 100644 --- a/drivers/infiniband/hw/hfi1/user_pages.c +++ b/drivers/infiniband/hw/hfi1/user_pages.c @@ -118,13 +118,10 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np void hfi1_release_user_pages(struct mm_struct *mm, struct page **p, size_t npages, bool dirty) { - size_t i; - - for (i = 0; i < npages; i++) { - if (dirty) - set_page_dirty_lock(p[i]); - put_page(p[i]); - } + if (dirty) + put_user_pages_dirty_lock(p, npages); + else + put_user_pages(p, npages); if (mm) { /* during close after signal, mm can be NULL */ atomic64_sub(npages, &mm->pinned_vm); diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c index 8ff0e90d7564..edccfd6e178f 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.c +++ b/drivers/infiniband/hw/mthca/mthca_memfree.c @@ -482,7 +482,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); if (ret < 0) { - put_page(pages[0]); + put_user_page(pages[0]); goto out; } @@ -490,7 +490,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, mthca_uarc_virt(dev, uar, i)); if (ret) { pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(sg_page(&db_tab->page[i].mem)); + put_user_page(sg_page(&db_tab->page[i].mem)); goto out; } @@ -556,7 +556,7 @@ void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, if (db_tab->page[i].uvirt) { mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1); pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(sg_page(&db_tab->page[i].mem)); + put_user_page(sg_page(&db_tab->page[i].mem)); } } diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c index f712fb7fa82f..bfbfbb7e0ff4 100644 --- a/drivers/infiniband/hw/qib/qib_user_pages.c +++ b/drivers/infiniband/hw/qib/qib_user_pages.c @@ -40,13 +40,10 @@ static void __qib_release_user_pages(struct page **p, size_t num_pages, int dirty) { - size_t i; - - for (i = 0; i < num_pages; i++) { - if (dirty) - set_page_dirty_lock(p[i]); - put_page(p[i]); - } + if (dirty) + put_user_pages_dirty_lock(p, num_pages); + else + put_user_pages(p, num_pages); } /** diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c index 0c204776263f..ac5bdb02144f 100644 --- a/drivers/infiniband/hw/qib/qib_user_sdma.c +++ b/drivers/infiniband/hw/qib/qib_user_sdma.c @@ -317,7 +317,7 @@ static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd, * the caller can ignore this page. */ if (put) { - put_page(page); + put_user_page(page); } else { /* coalesce case */ kunmap(page); @@ -631,7 +631,7 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev, kunmap(pkt->addr[i].page); if (pkt->addr[i].put_page) - put_page(pkt->addr[i].page); + put_user_page(pkt->addr[i].page); else __free_page(pkt->addr[i].page); } else if (pkt->addr[i].kvaddr) { @@ -706,7 +706,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd, /* if error, return all pages not managed by pkt */ free_pages: while (i < j) - put_page(pages[i++]); + put_user_page(pages[i++]); done: return ret; diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index e312f522a66d..0b0237d41613 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -75,9 +75,10 @@ static void usnic_uiom_put_pages(struct list_head *chunk_list, int dirty) for_each_sg(chunk->page_list, sg, chunk->nents, i) { page = sg_page(sg); pa = sg_phys(sg); - if (!PageDirty(page) && dirty) - set_page_dirty_lock(page); - put_page(page); + if (dirty) + put_user_pages_dirty_lock(&page, 1); + else + put_user_page(page); usnic_dbg("pa: %pa\n", &pa); } kfree(chunk); -- cgit v1.2.3 From 5f5e4eb4fb2d6c09db18dc431a0c4a3b11ff5bae Mon Sep 17 00:00:00 2001 From: Dennis Dalessandro Date: Fri, 24 May 2019 11:44:58 -0400 Subject: IB/hfi1: Remove extra brackets from an if A recent patch to hfi1 left behind a checkpatch error. Fixes: fb24ea52f78e ("drivers: Remove explicit invocations of mmiowb()") Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/pio.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index 16ba9d52e1b9..919008620dd3 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -1577,9 +1577,8 @@ void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint) else sc_del_credit_return_intr(sc); trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl); - if (needint) { + if (needint) sc_return_credits(sc); - } } /** -- cgit v1.2.3 From 255efcaeb623dd9777523b98d29aa5a0792d9245 Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Tue, 28 May 2019 15:46:14 +0300 Subject: RDMA/efa: Use kvzalloc instead of kzalloc with fallback Use kvzalloc which attempts to allocate a physically continuous buffer and fallbacks to virtually continuous on failure instead of open coding it in the driver. The is_vmalloc_addr function is used to determine whether the buffer is physically continuous or not (which determines direct vs indirect MR registration mode). Suggested-by: Jason Gunthorpe Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 46 ++++++++++++++++------------------- 1 file changed, 21 insertions(+), 25 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 4999a74cee24..b791cc90d106 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1285,30 +1285,30 @@ static int pbl_create(struct efa_dev *dev, int err; pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE; - pbl->pbl_buf = kzalloc(pbl->pbl_buf_size_in_bytes, - GFP_KERNEL | __GFP_NOWARN); - if (pbl->pbl_buf) { - pbl->physically_continuous = 1; + pbl->pbl_buf = kvzalloc(pbl->pbl_buf_size_in_bytes, GFP_KERNEL); + if (!pbl->pbl_buf) + return -ENOMEM; + + if (is_vmalloc_addr(pbl->pbl_buf)) { + pbl->physically_continuous = 0; err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, hp_shift); if (err) - goto err_continuous; - err = pbl_continuous_initialize(dev, pbl); + goto err_free; + + err = pbl_indirect_initialize(dev, pbl); if (err) - goto err_continuous; + goto err_free; } else { - pbl->physically_continuous = 0; - pbl->pbl_buf = vzalloc(pbl->pbl_buf_size_in_bytes); - if (!pbl->pbl_buf) - return -ENOMEM; - + pbl->physically_continuous = 1; err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, hp_shift); if (err) - goto err_indirect; - err = pbl_indirect_initialize(dev, pbl); + goto err_free; + + err = pbl_continuous_initialize(dev, pbl); if (err) - goto err_indirect; + goto err_free; } ibdev_dbg(&dev->ibdev, @@ -1317,24 +1317,20 @@ static int pbl_create(struct efa_dev *dev, return 0; -err_continuous: - kfree(pbl->pbl_buf); - return err; -err_indirect: - vfree(pbl->pbl_buf); +err_free: + kvfree(pbl->pbl_buf); return err; } static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl) { - if (pbl->physically_continuous) { + if (pbl->physically_continuous) dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr, pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); - kfree(pbl->pbl_buf); - } else { + else pbl_indirect_terminate(dev, pbl); - vfree(pbl->pbl_buf); - } + + kvfree(pbl->pbl_buf); } static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr, -- cgit v1.2.3 From e0e3f39759151fb1b445a0dbc5d6a14f3e1732aa Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Tue, 28 May 2019 15:46:15 +0300 Subject: RDMA/efa: Remove unneeded admin commands abort flow The admin commands abort flow is buggy (use-after-free) and not really necessary as it is guaranteed that after ib_unregister_device() is called there are no user verbs threads running in parallel, delete it. Suggested-by: Jason Gunthorpe Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_com.c | 74 +------------------------------------ drivers/infiniband/hw/efa/efa_com.h | 1 - 2 files changed, 1 insertion(+), 74 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index a5c788741a04..ec04ced9fd2b 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -39,8 +39,6 @@ enum efa_cmd_status { EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, - /* Abort - canceled by the driver */ - EFA_CMD_ABORTED, }; struct efa_comp_ctx { @@ -532,16 +530,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c msleep(aq->poll_interval); } - if (comp_ctx->status == EFA_CMD_ABORTED) { - ibdev_err(aq->efa_dev, "Command was aborted\n"); - atomic64_inc(&aq->stats.aborted_cmd); - err = -ENODEV; - goto out; - } - - WARN_ONCE(comp_ctx->status != EFA_CMD_COMPLETED, - "Invalid completion status %d\n", comp_ctx->status); - err = efa_com_comp_status_to_errno(comp_ctx->comp_status); out: efa_com_put_comp_ctx(aq, comp_ctx); @@ -665,66 +653,6 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, return err; } -/** - * efa_com_abort_admin_commands - Abort all the outstanding admin commands. - * @edev: EFA communication layer struct - * - * This method aborts all the outstanding admin commands. - * The caller should then call efa_com_wait_for_abort_completion to make sure - * all the commands were completed. - */ -static void efa_com_abort_admin_commands(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - struct efa_comp_ctx *comp_ctx; - unsigned long flags; - u16 i; - - spin_lock(&aq->sq.lock); - spin_lock_irqsave(&aq->cq.lock, flags); - for (i = 0; i < aq->depth; i++) { - comp_ctx = efa_com_get_comp_ctx(aq, i, false); - if (!comp_ctx) - break; - - comp_ctx->status = EFA_CMD_ABORTED; - - complete(&comp_ctx->wait_event); - } - spin_unlock_irqrestore(&aq->cq.lock, flags); - spin_unlock(&aq->sq.lock); -} - -/** - * efa_com_wait_for_abort_completion - Wait for admin commands abort. - * @edev: EFA communication layer struct - * - * This method wait until all the outstanding admin commands will be completed. - */ -static void efa_com_wait_for_abort_completion(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - int i; - - /* all mine */ - for (i = 0; i < aq->depth; i++) - down(&aq->avail_cmds); - - /* let it go */ - for (i = 0; i < aq->depth; i++) - up(&aq->avail_cmds); -} - -static void efa_com_admin_flush(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - - clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - - efa_com_abort_admin_commands(edev); - efa_com_wait_for_abort_completion(edev); -} - /** * efa_com_admin_destroy - Destroy the admin and the async events queues. * @edev: EFA communication layer struct @@ -737,7 +665,7 @@ void efa_com_admin_destroy(struct efa_com_dev *edev) struct efa_com_admin_sq *sq = &aq->sq; u16 size; - efa_com_admin_flush(edev); + clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); devm_kfree(edev->dmadev, aq->comp_ctx_pool); devm_kfree(edev->dmadev, aq->comp_ctx); diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index 84d96724a74b..c67dd8109d1c 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -45,7 +45,6 @@ struct efa_com_admin_sq { /* Don't use anything other than atomic64 */ struct efa_com_stats_admin { - atomic64_t aborted_cmd; atomic64_t submitted_cmd; atomic64_t completed_cmd; atomic64_t no_completion; -- cgit v1.2.3 From 4d50e084c560fa952e235d092cf53c86d8bf4b7b Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Tue, 28 May 2019 15:46:17 +0300 Subject: RDMA/efa: Use rdma block iterator in chunk list creation When creating the chunks list the rdma_for_each_block() iterator is used in order to iterate over the payload in EFA_CHUNK_PAYLOAD_SIZE (device defined) strides. Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Reviewed-by: Shiraz Saleem Signed-off-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index b791cc90d106..607aff869200 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1085,14 +1085,14 @@ err: */ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) { - unsigned int entry, payloads_in_sg, chunk_list_size, chunk_idx, payload_idx; struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages; struct scatterlist *pages_sgl = pbl->phys.indirect.sgl; + unsigned int chunk_list_size, chunk_idx, payload_idx; int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt; struct efa_com_ctrl_buff_info *ctrl_buf; u64 *cur_chunk_buf, *prev_chunk_buf; - struct scatterlist *sg; + struct ib_block_iter biter; dma_addr_t dma_addr; int i; @@ -1126,18 +1126,15 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) chunk_idx = 0; payload_idx = 0; cur_chunk_buf = chunk_list->chunks[0].buf; - for_each_sg(pages_sgl, sg, sg_dma_cnt, entry) { - payloads_in_sg = sg_dma_len(sg) >> EFA_CHUNK_PAYLOAD_SHIFT; - for (i = 0; i < payloads_in_sg; i++) { - cur_chunk_buf[payload_idx++] = - (sg_dma_address(sg) & ~(EFA_CHUNK_PAYLOAD_SIZE - 1)) + - (EFA_CHUNK_PAYLOAD_SIZE * i); - - if (payload_idx == EFA_PTRS_PER_CHUNK) { - chunk_idx++; - cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; - payload_idx = 0; - } + rdma_for_each_block(pages_sgl, &biter, sg_dma_cnt, + EFA_CHUNK_PAYLOAD_SIZE) { + cur_chunk_buf[payload_idx++] = + rdma_block_iter_dma_address(&biter); + + if (payload_idx == EFA_PTRS_PER_CHUNK) { + chunk_idx++; + cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; + payload_idx = 0; } } -- cgit v1.2.3 From 2367d00e2ca3dfda1b30f89d854cb1669b941e7f Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Tue, 28 May 2019 15:46:18 +0300 Subject: RDMA/efa: Remove unused includes Remove leftover includes that are no longer used from the driver. Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa.h | 2 -- drivers/infiniband/hw/efa/efa_com_cmd.c | 1 - 2 files changed, 3 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 9e3cc3239c13..14a36546985b 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -7,10 +7,8 @@ #define _EFA_H_ #include -#include #include #include -#include #include #include diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 14227725521c..91e7f2195802 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -3,7 +3,6 @@ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. */ -#include "efa.h" #include "efa_com.h" #include "efa_com_cmd.h" -- cgit v1.2.3 From 829ca44ecf60e9b6f83d0161a6ef10c1304c5060 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 29 May 2019 10:13:26 -0500 Subject: IB/qib: Use struct_size() helper Make use of the struct_size() helper instead of an open-coded version in order to avoid any potential type mistakes, in particular in the context in which this code is being used. So, replace the following form: sizeof(*pkt) + sizeof(pkt->addr[0])*n with: struct_size(pkt, addr, n) Also, notice that variable size is unnecessary, hence it is removed. This code was detected with the help of Coccinelle. Signed-off-by: Gustavo A. R. Silva Reviewed-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/qib/qib_user_sdma.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c index ac5bdb02144f..05190edc2611 100644 --- a/drivers/infiniband/hw/qib/qib_user_sdma.c +++ b/drivers/infiniband/hw/qib/qib_user_sdma.c @@ -904,10 +904,11 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, } if (frag_size) { - int pktsize, tidsmsize, n; + int tidsmsize, n; + size_t pktsize; n = npages*((2*PAGE_SIZE/frag_size)+1); - pktsize = sizeof(*pkt) + sizeof(pkt->addr[0])*n; + pktsize = struct_size(pkt, addr, n); /* * Determine if this is tid-sdma or just sdma. -- cgit v1.2.3 From 6fe1a9b9b6542d460099395492d029bb6c6c2f5e Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Wed, 29 May 2019 10:15:28 -0500 Subject: IB/hfi1: Use struct_size() helper Make use of the struct_size() helper instead of an open-coded version in order to avoid any potential type mistakes, in particular in the context in which this code is being used. So, replace the following form: sizeof(struct opa_port_status_rsp) + num_vls * sizeof(struct _vls_pctrs) with: struct_size(rsp, vls, num_vls) and so on... Also, notice that variable size is unnecessary, hence it is removed. This code was detected with the help of Coccinelle. Signed-off-by: Gustavo A. R. Silva Reviewed-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/mad.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index 4228393e6c4c..184dba3c2828 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -2744,8 +2744,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, u16 link_width; u16 link_speed; - response_data_size = sizeof(struct opa_port_status_rsp) + - num_vls * sizeof(struct _vls_pctrs); + response_data_size = struct_size(rsp, vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= OPA_PM_STATUS_REQUEST_TOO_LARGE; return reply((struct ib_mad_hdr *)pmp); @@ -3014,8 +3013,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, } /* Sanity check */ - response_data_size = sizeof(struct opa_port_data_counters_msg) + - num_vls * sizeof(struct _vls_dctrs); + response_data_size = struct_size(req, port[0].vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD; @@ -3232,8 +3230,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, return reply((struct ib_mad_hdr *)pmp); } - response_data_size = sizeof(struct opa_port_error_counters64_msg) + - num_vls * sizeof(struct _vls_ectrs); + response_data_size = struct_size(req, port[0].vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD; -- cgit v1.2.3 From 97545b10221ad14b046dba135a37f4e98a560697 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Thu, 30 May 2019 23:55:53 +0800 Subject: RDMA/hns: Bugfix for posting multiple srq work request When the user submits more than 32 work request to a srq queue at a time, it needs to find the corresponding number of entries in the bitmap in the idx queue. However, the original lookup function named ffs only processes 32 bits of the array element, When the number of srq wqe issued exceeds 32, the ffs will only process the lower 32 bits of the elements, it will not be able to get the correct wqe index for srq wqe. Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_device.h | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 34 +++++++++++++++-------------- drivers/infiniband/hw/hns/hns_roce_srq.c | 15 +++---------- 3 files changed, 22 insertions(+), 29 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index d6e8b446fc57..ce23338831eb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -489,7 +489,7 @@ struct hns_roce_idx_que { u32 buf_size; struct ib_umem *umem; struct hns_roce_mtt mtt; - u64 *bitmap; + unsigned long *bitmap; }; struct hns_roce_srq { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 7fcec998618a..ac017c24b200 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2340,15 +2340,10 @@ static void *get_srq_wqe(struct hns_roce_srq *srq, int n) static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) { - u32 bitmap_num; - int bit_num; - /* always called with interrupts disabled. */ spin_lock(&srq->lock); - bitmap_num = wqe_index / (sizeof(u64) * 8); - bit_num = wqe_index % (sizeof(u64) * 8); - srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num); + bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); srq->tail++; spin_unlock(&srq->lock); @@ -5977,18 +5972,19 @@ out: return ret; } -static int find_empty_entry(struct hns_roce_idx_que *idx_que) +static int find_empty_entry(struct hns_roce_idx_que *idx_que, + unsigned long size) { - int bit_num; - int i; + int wqe_idx; - /* bitmap[i] is set zero if all bits are allocated */ - for (i = 0; idx_que->bitmap[i] == 0; ++i) - ; - bit_num = ffs(idx_que->bitmap[i]); - idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1)); + if (unlikely(bitmap_full(idx_que->bitmap, size))) + return -ENOSPC; + + wqe_idx = find_first_zero_bit(idx_que->bitmap, size); + + bitmap_set(idx_que->bitmap, wqe_idx, 1); - return i * BITS_PER_LONG_LONG + (bit_num - 1); + return wqe_idx; } static void fill_idx_queue(struct hns_roce_idx_que *idx_que, @@ -6034,7 +6030,13 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, break; } - wqe_idx = find_empty_entry(&srq->idx_que); + wqe_idx = find_empty_entry(&srq->idx_que, srq->max); + if (wqe_idx < 0) { + ret = -ENOMEM; + *bad_wr = wr; + break; + } + fill_idx_queue(&srq->idx_que, ind, wqe_idx); wqe = get_srq_wqe(srq, wqe_idx); dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index ad15b41da30a..c222f243953a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -181,28 +181,19 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq, { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); struct hns_roce_idx_que *idx_que = &srq->idx_que; - u32 bitmap_num; - int i; - bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64)); - - idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL); + idx_que->bitmap = bitmap_zalloc(srq->max, GFP_KERNEL); if (!idx_que->bitmap) return -ENOMEM; - bitmap_num = bitmap_num / (8 * sizeof(u64)); - idx_que->buf_size = srq->idx_que.buf_size; if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2, &idx_que->idx_buf, page_shift)) { - kfree(idx_que->bitmap); + bitmap_free(idx_que->bitmap); return -ENOMEM; } - for (i = 0; i < bitmap_num; i++) - idx_que->bitmap[i] = ~(0UL); - return 0; } @@ -395,7 +386,7 @@ err_idx_mtt: err_create_idx: hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, &srq->idx_que.idx_buf); - kfree(srq->idx_que.bitmap); + bitmap_free(srq->idx_que.bitmap); err_srq_mtt: hns_roce_mtt_cleanup(hr_dev, &srq->mtt); -- cgit v1.2.3 From fa027328a1c93d72031e6cc6b4a7eee967fd7406 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 31 May 2019 10:21:01 +0100 Subject: RDMA/hns: fix inverted logic of readl read and shift A previous change incorrectly changed the inverted logic and logically negated the readl rather than the shifted readl result. Fix this by adding in missing parentheses around the expression that needs to be logically negated. Addresses-Coverity: ("Logically dead code") Fixes: 669cefb654cb ("RDMA/hns: Remove jiffies operation in disable interrupt context") Signed-off-by: Colin Ian King Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 157c84a1f55f..8490a86c3ef0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -378,7 +378,7 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, end = HW_SYNC_TIMEOUT_MSECS; while (end) { - if (!readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) + if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) break; mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); -- cgit v1.2.3 From 4f18904c78495ec16e271ee507709c626e61a62a Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Fri, 31 May 2019 18:28:03 +0800 Subject: RDMA/hns: Bugfix for filling the sge of srq When user post recv a srq with multiple sges, the hardware will get the last correct sge and count the sge numbers according to the specific identifier with lkey. For example, when the driver fills the sges with every wr less than the max sge that the user configured when creating srq, the hardware will stop getting the sge according to the specific lkey in the sge. However, it will always end with the first sge in the current post srq recv interface implementation. Fixes: c7bcb13442e1 ("RDMA/hns: Add SRQ support for hip08 kernel mode") Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index ac017c24b200..2d27dc91a823 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6048,9 +6048,9 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, } if (i < srq->max_gs) { - dseg->len = 0; - dseg->lkey = cpu_to_le32(0x100); - dseg->addr = 0; + dseg[i].len = 0; + dseg[i].lkey = cpu_to_le32(0x100); + dseg[i].addr = 0; } srq->wrid[wqe_idx] = wr->wr_id; -- cgit v1.2.3 From b9560a419bfd498279333387817adcf5faef2825 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 5 Jun 2019 14:39:24 -0300 Subject: RDMA: Move driver_id into struct ib_device_ops No reason for every driver to emit code to set this, just make it part of the driver's existing static const ops structure. Signed-off-by: Jason Gunthorpe Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/main.c | 3 ++- drivers/infiniband/hw/cxgb3/iwch_provider.c | 3 ++- drivers/infiniband/hw/cxgb4/provider.c | 3 ++- drivers/infiniband/hw/efa/efa_main.c | 3 ++- drivers/infiniband/hw/hfi1/verbs.c | 4 +++- drivers/infiniband/hw/hns/hns_roce_main.c | 3 ++- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 3 ++- drivers/infiniband/hw/mlx4/main.c | 3 ++- drivers/infiniband/hw/mlx5/main.c | 3 ++- drivers/infiniband/hw/mthca/mthca_provider.c | 3 ++- drivers/infiniband/hw/nes/nes_verbs.c | 3 ++- drivers/infiniband/hw/ocrdma/ocrdma_main.c | 3 ++- drivers/infiniband/hw/qedr/main.c | 3 ++- drivers/infiniband/hw/qib/qib_verbs.c | 4 +++- drivers/infiniband/hw/usnic/usnic_ib_main.c | 3 ++- drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c | 3 ++- 16 files changed, 34 insertions(+), 16 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 814f959c7db9..1ef5f83ec914 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -596,6 +596,8 @@ static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) } static const struct ib_device_ops bnxt_re_dev_ops = { + .driver_id = RDMA_DRIVER_BNXT_RE, + .add_gid = bnxt_re_add_gid, .alloc_hw_stats = bnxt_re_ib_alloc_hw_stats, .alloc_mr = bnxt_re_alloc_mr, @@ -691,7 +693,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group); - ibdev->driver_id = RDMA_DRIVER_BNXT_RE; ib_set_device_ops(ibdev, &bnxt_re_dev_ops); ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1); if (ret) diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 4bfab739ec0d..9a9527fdcb9a 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -1236,6 +1236,8 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str) } static const struct ib_device_ops iwch_dev_ops = { + .driver_id = RDMA_DRIVER_CXGB3, + .alloc_hw_stats = iwch_alloc_stats, .alloc_mr = iwch_alloc_mr, .alloc_mw = iwch_alloc_mw, @@ -1319,7 +1321,6 @@ int iwch_register_device(struct iwch_dev *dev) memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name, sizeof(dev->ibdev.iw_ifname)); - dev->ibdev.driver_id = RDMA_DRIVER_CXGB3; rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group); ib_set_device_ops(&dev->ibdev, &iwch_dev_ops); return ib_register_device(&dev->ibdev, "cxgb3_%d"); diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 8ed75b141521..74644afe25ab 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -489,6 +489,8 @@ static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res) } static const struct ib_device_ops c4iw_dev_ops = { + .driver_id = RDMA_DRIVER_CXGB4, + .alloc_hw_stats = c4iw_alloc_stats, .alloc_mr = c4iw_alloc_mr, .alloc_mw = c4iw_alloc_mw, @@ -599,7 +601,6 @@ void c4iw_register_device(struct work_struct *work) sizeof(dev->ibdev.iw_ifname)); rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group); - dev->ibdev.driver_id = RDMA_DRIVER_CXGB4; ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops); ret = set_netdevs(&dev->ibdev, &dev->rdev); if (ret) diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index db974caf1eb1..3803dd4526b5 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -197,6 +197,8 @@ static void efa_stats_init(struct efa_dev *dev) } static const struct ib_device_ops efa_dev_ops = { + .driver_id = RDMA_DRIVER_EFA, + .alloc_pd = efa_alloc_pd, .alloc_ucontext = efa_alloc_ucontext, .create_ah = efa_create_ah, @@ -287,7 +289,6 @@ static int efa_ib_device_add(struct efa_dev *dev) dev->ibdev.uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE); - dev->ibdev.driver_id = RDMA_DRIVER_EFA; ib_set_device_ops(&dev->ibdev, &efa_dev_ops); err = ib_register_device(&dev->ibdev, "efa_%d"); diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index 1eb4105b2d22..a97f4f9e5c6a 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1779,6 +1779,8 @@ static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, } static const struct ib_device_ops hfi1_dev_ops = { + .driver_id = RDMA_DRIVER_HFI1, + .alloc_hw_stats = alloc_hw_stats, .alloc_rdma_netdev = hfi1_vnic_alloc_rn, .get_dev_fw_str = hfi1_get_dev_fw_str, @@ -1923,7 +1925,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &ib_hfi1_attr_group); - ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_HFI1); + ret = rvt_register_device(&dd->verbs_dev.rdi); if (ret) goto err_verbs_txreq; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index a6c5c67d0b87..dd408f8afe72 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -414,6 +414,8 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) } static const struct ib_device_ops hns_roce_dev_ops = { + .driver_id = RDMA_DRIVER_HNS, + .add_gid = hns_roce_add_gid, .alloc_pd = hns_roce_alloc_pd, .alloc_ucontext = hns_roce_alloc_ucontext, @@ -536,7 +538,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); } - ib_dev->driver_id = RDMA_DRIVER_HNS; ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); ib_set_device_ops(ib_dev, &hns_roce_dev_ops); for (i = 0; i < hr_dev->caps.num_ports; i++) { diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index a10a30d44b32..1979cefdf90c 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -2650,6 +2650,8 @@ static int i40iw_query_pkey(struct ib_device *ibdev, } static const struct ib_device_ops i40iw_dev_ops = { + .driver_id = RDMA_DRIVER_I40IW, + .alloc_hw_stats = i40iw_alloc_hw_stats, .alloc_mr = i40iw_alloc_mr, .alloc_pd = i40iw_alloc_pd, @@ -2787,7 +2789,6 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev) return -ENOMEM; iwibdev = iwdev->iwibdev; rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group); - iwibdev->ibdev.driver_id = RDMA_DRIVER_I40IW; ret = ib_register_device(&iwibdev->ibdev, "i40iw%d"); if (ret) goto error; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 25d09d53b51c..03847e2f7835 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -2509,6 +2509,8 @@ static void get_fw_ver_str(struct ib_device *device, char *str) } static const struct ib_device_ops mlx4_ib_dev_ops = { + .driver_id = RDMA_DRIVER_MLX4, + .add_gid = mlx4_ib_add_gid, .alloc_mr = mlx4_ib_alloc_mr, .alloc_pd = mlx4_ib_alloc_pd, @@ -2839,7 +2841,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_steer_free_bitmap; rdma_set_device_sysfs_group(&ibdev->ib_dev, &mlx4_attr_group); - ibdev->ib_dev.driver_id = RDMA_DRIVER_MLX4; if (ib_register_device(&ibdev->ib_dev, "mlx4_%d")) goto err_diag_counters; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index abac70ad5c7c..abd416a31e71 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6124,6 +6124,8 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) } static const struct ib_device_ops mlx5_ib_dev_ops = { + .driver_id = RDMA_DRIVER_MLX5, + .add_gid = mlx5_ib_add_gid, .alloc_mr = mlx5_ib_alloc_mr, .alloc_pd = mlx5_ib_alloc_pd, @@ -6290,7 +6292,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops); - dev->ib_dev.driver_id = RDMA_DRIVER_MLX5; ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops); if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 4f40dfedf920..d6467da39aab 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -1153,6 +1153,8 @@ static void get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops mthca_dev_ops = { + .driver_id = RDMA_DRIVER_MTHCA, + .alloc_pd = mthca_alloc_pd, .alloc_ucontext = mthca_alloc_ucontext, .attach_mcast = mthca_multicast_attach, @@ -1303,7 +1305,6 @@ int mthca_register_device(struct mthca_dev *dev) mutex_init(&dev->cap_mask_mutex); rdma_set_device_sysfs_group(&dev->ib_dev, &mthca_attr_group); - dev->ib_dev.driver_id = RDMA_DRIVER_MTHCA; ret = ib_register_device(&dev->ib_dev, "mthca%d"); if (ret) return ret; diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index fb2d0762c7c8..3c85e2ef5a08 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -3558,6 +3558,8 @@ static void get_dev_fw_str(struct ib_device *dev, char *str) } static const struct ib_device_ops nes_dev_ops = { + .driver_id = RDMA_DRIVER_NES, + .alloc_mr = nes_alloc_mr, .alloc_mw = nes_alloc_mw, .alloc_pd = nes_alloc_pd, @@ -3722,7 +3724,6 @@ int nes_register_ofa_device(struct nes_ib_device *nesibdev) int ret; rdma_set_device_sysfs_group(&nesvnic->nesibdev->ibdev, &nes_attr_group); - nesvnic->nesibdev->ibdev.driver_id = RDMA_DRIVER_NES; ret = ib_register_device(&nesvnic->nesibdev->ibdev, "nes%d"); if (ret) { return ret; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index fc6c0962dea9..a9da4b857566 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -144,6 +144,8 @@ static const struct attribute_group ocrdma_attr_group = { }; static const struct ib_device_ops ocrdma_dev_ops = { + .driver_id = RDMA_DRIVER_OCRDMA, + .alloc_mr = ocrdma_alloc_mr, .alloc_pd = ocrdma_alloc_pd, .alloc_ucontext = ocrdma_alloc_ucontext, @@ -249,7 +251,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) ib_set_device_ops(&dev->ibdev, &ocrdma_dev_srq_ops); } rdma_set_device_sysfs_group(&dev->ibdev, &ocrdma_attr_group); - dev->ibdev.driver_id = RDMA_DRIVER_OCRDMA; ret = ib_device_set_netdev(&dev->ibdev, dev->nic_info.netdev, 1); if (ret) return ret; diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 083c2c00a8e9..737745231f8f 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -183,6 +183,8 @@ static void qedr_roce_register_device(struct qedr_dev *dev) } static const struct ib_device_ops qedr_dev_ops = { + .driver_id = RDMA_DRIVER_QEDR, + .alloc_mr = qedr_alloc_mr, .alloc_pd = qedr_alloc_pd, .alloc_ucontext = qedr_alloc_ucontext, @@ -274,7 +276,6 @@ static int qedr_register_device(struct qedr_dev *dev) rdma_set_device_sysfs_group(&dev->ibdev, &qedr_attr_group); ib_set_device_ops(&dev->ibdev, &qedr_dev_ops); - dev->ibdev.driver_id = RDMA_DRIVER_QEDR; rc = ib_device_set_netdev(&dev->ibdev, dev->ndev, 1); if (rc) return rc; diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index 5ff32d32c61c..bbc331d7f49b 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -1482,6 +1482,8 @@ static void qib_fill_device_attr(struct qib_devdata *dd) } static const struct ib_device_ops qib_dev_ops = { + .driver_id = RDMA_DRIVER_QIB, + .init_port = qib_create_port_files, .modify_device = qib_modify_device, .process_mad = qib_process_mad, @@ -1616,7 +1618,7 @@ int qib_register_ib_device(struct qib_devdata *dd) rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &qib_attr_group); ib_set_device_ops(ibdev, &qib_dev_ops); - ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_QIB); + ret = rvt_register_device(&dd->verbs_dev.rdi); if (ret) goto err_tx; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index d88d9f8a7f9a..47830334cb55 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -329,6 +329,8 @@ static void usnic_get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops usnic_dev_ops = { + .driver_id = RDMA_DRIVER_USNIC, + .alloc_pd = usnic_ib_alloc_pd, .alloc_ucontext = usnic_ib_alloc_ucontext, .create_cq = usnic_ib_create_cq, @@ -412,7 +414,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev) ib_set_device_ops(&us_ibdev->ib_dev, &usnic_dev_ops); - us_ibdev->ib_dev.driver_id = RDMA_DRIVER_USNIC; rdma_set_device_sysfs_group(&us_ibdev->ib_dev, &usnic_attr_group); ret = ib_device_set_netdev(&us_ibdev->ib_dev, us_ibdev->netdev, 1); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 40182297f87f..54a0b6372629 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -144,6 +144,8 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num, } static const struct ib_device_ops pvrdma_dev_ops = { + .driver_id = RDMA_DRIVER_VMW_PVRDMA, + .add_gid = pvrdma_add_gid, .alloc_mr = pvrdma_alloc_mr, .alloc_pd = pvrdma_alloc_pd, @@ -261,7 +263,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) if (!dev->srq_tbl) goto err_qp_free; } - dev->ib_dev.driver_id = RDMA_DRIVER_VMW_PVRDMA; ret = ib_device_set_netdev(&dev->ib_dev, dev->netdev, 1); if (ret) return ret; -- cgit v1.2.3 From 72c6ec18eb6161c8fc672ae96ec5c77df4d07405 Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 5 Jun 2019 14:39:25 -0300 Subject: RDMA: Move uverbs_abi_ver into struct ib_device_ops No reason for every driver to emit code to set this, just make it part of the driver's existing static const ops structure. Signed-off-by: Jason Gunthorpe Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 6 +++--- drivers/infiniband/hw/bnxt_re/main.c | 2 +- drivers/infiniband/hw/cxgb3/iwch_provider.c | 2 +- drivers/infiniband/hw/cxgb4/provider.c | 2 +- drivers/infiniband/hw/efa/efa_main.c | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 2 +- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2 ++ drivers/infiniband/hw/mlx4/main.c | 15 ++++++++------- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mthca/mthca_provider.c | 2 +- drivers/infiniband/hw/nes/nes_verbs.c | 2 ++ drivers/infiniband/hw/ocrdma/ocrdma_main.c | 2 +- drivers/infiniband/hw/qedr/main.c | 2 +- drivers/infiniband/hw/usnic/usnic_ib_main.c | 2 +- drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c | 2 +- 15 files changed, 26 insertions(+), 21 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 2c3685faa57a..8af8e1472101 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3630,10 +3630,10 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) u32 chip_met_rev_num = 0; int rc; - dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", - ibdev->uverbs_abi_ver); + dev_dbg(rdev_to_dev(rdev), "ABI version requested %u", + ibdev->ops.uverbs_abi_ver); - if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { + if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", BNXT_RE_ABI_VERSION); return -EPERM; diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 1ef5f83ec914..a45cb9ee51ab 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -597,6 +597,7 @@ static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) static const struct ib_device_ops bnxt_re_dev_ops = { .driver_id = RDMA_DRIVER_BNXT_RE, + .uverbs_abi_ver = BNXT_RE_ABI_VERSION, .add_gid = bnxt_re_add_gid, .alloc_hw_stats = bnxt_re_ib_alloc_hw_stats, @@ -663,7 +664,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; /* User space */ - ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 9a9527fdcb9a..5b410605a3a1 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -1237,6 +1237,7 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str) static const struct ib_device_ops iwch_dev_ops = { .driver_id = RDMA_DRIVER_CXGB3, + .uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION, .alloc_hw_stats = iwch_alloc_stats, .alloc_mr = iwch_alloc_mr, @@ -1316,7 +1317,6 @@ int iwch_register_device(struct iwch_dev *dev) dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports; dev->ibdev.num_comp_vectors = 1; dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev; - dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION; memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name, sizeof(dev->ibdev.iw_ifname)); diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 74644afe25ab..c56cdfbd8a88 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -490,6 +490,7 @@ static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res) static const struct ib_device_ops c4iw_dev_ops = { .driver_id = RDMA_DRIVER_CXGB4, + .uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION, .alloc_hw_stats = c4iw_alloc_stats, .alloc_mr = c4iw_alloc_mr, @@ -595,7 +596,6 @@ void c4iw_register_device(struct work_struct *work) dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports; dev->ibdev.num_comp_vectors = dev->rdev.lldi.nciq; dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev; - dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION; memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name, sizeof(dev->ibdev.iw_ifname)); diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 3803dd4526b5..b05c5a0b9bc0 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -198,6 +198,7 @@ static void efa_stats_init(struct efa_dev *dev) static const struct ib_device_ops efa_dev_ops = { .driver_id = RDMA_DRIVER_EFA, + .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION, .alloc_pd = efa_alloc_pd, .alloc_ucontext = efa_alloc_ucontext, @@ -266,7 +267,6 @@ static int efa_ib_device_add(struct efa_dev *dev) dev->ibdev.phys_port_cnt = 1; dev->ibdev.num_comp_vectors = 1; dev->ibdev.dev.parent = &pdev->dev; - dev->ibdev.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index dd408f8afe72..e496b0628e25 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -415,6 +415,7 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) static const struct ib_device_ops hns_roce_dev_ops = { .driver_id = RDMA_DRIVER_HNS, + .uverbs_abi_ver = 1, .add_gid = hns_roce_add_gid, .alloc_pd = hns_roce_alloc_pd, @@ -489,7 +490,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_dev->phys_port_cnt = hr_dev->caps.num_ports; ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; - ib_dev->uverbs_abi_ver = 1; ib_dev->uverbs_cmd_mask = (1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 1979cefdf90c..4dc647c8556b 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -2651,6 +2651,8 @@ static int i40iw_query_pkey(struct ib_device *ibdev, static const struct ib_device_ops i40iw_dev_ops = { .driver_id = RDMA_DRIVER_I40IW, + /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ + .uverbs_abi_ver = I40IW_ABI_VER, .alloc_hw_stats = i40iw_alloc_hw_stats, .alloc_mr = i40iw_alloc_mr, diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 03847e2f7835..1f87221acbb0 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1089,7 +1089,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, if (!dev->ib_active) return -EAGAIN; - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { + if (ibdev->ops.uverbs_abi_ver == + MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { resp_v3.qp_tab_size = dev->dev->caps.num_qps; resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size; resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; @@ -1111,7 +1112,7 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, INIT_LIST_HEAD(&context->wqn_ranges_list); mutex_init(&context->wqn_ranges_mutex); - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) + if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); else err = ib_copy_to_udata(udata, &resp, sizeof(resp)); @@ -2510,6 +2511,7 @@ static void get_fw_ver_str(struct ib_device *device, char *str) static const struct ib_device_ops mlx4_ib_dev_ops = { .driver_id = RDMA_DRIVER_MLX4, + .uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION, .add_gid = mlx4_ib_add_gid, .alloc_mr = mlx4_ib_alloc_mr, @@ -2653,11 +2655,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; ibdev->ib_dev.dev.parent = &dev->persist->pdev->dev; - if (dev->caps.userspace_caps) - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; - else - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; - ibdev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -2731,6 +2728,10 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_fs_ops); } + if (!dev->caps.userspace_caps) + ibdev->ib_dev.ops.uverbs_abi_ver = + MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; + mlx4_ib_alloc_eqs(dev, ibdev); spin_lock_init(&iboe->lock); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index abd416a31e71..0c23eccb8855 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6125,6 +6125,7 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) static const struct ib_device_ops mlx5_ib_dev_ops = { .driver_id = RDMA_DRIVER_MLX5, + .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION, .add_gid = mlx5_ib_add_gid, .alloc_mr = mlx5_ib_alloc_mr, @@ -6223,7 +6224,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) struct mlx5_core_dev *mdev = dev->mdev; int err; - dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index d6467da39aab..690c65accea4 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -1154,6 +1154,7 @@ static void get_dev_fw_str(struct ib_device *device, char *str) static const struct ib_device_ops mthca_dev_ops = { .driver_id = RDMA_DRIVER_MTHCA, + .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, .alloc_pd = mthca_alloc_pd, .alloc_ucontext = mthca_alloc_ucontext, @@ -1247,7 +1248,6 @@ int mthca_register_device(struct mthca_dev *dev) dev->ib_dev.owner = THIS_MODULE; - dev->ib_dev.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 3c85e2ef5a08..f1fdd6829a40 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -3559,6 +3559,8 @@ static void get_dev_fw_str(struct ib_device *dev, char *str) static const struct ib_device_ops nes_dev_ops = { .driver_id = RDMA_DRIVER_NES, + /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ + .uverbs_abi_ver = NES_ABI_USERSPACE_VER, .alloc_mr = nes_alloc_mr, .alloc_mw = nes_alloc_mw, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index a9da4b857566..ef823f1144b5 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -145,6 +145,7 @@ static const struct attribute_group ocrdma_attr_group = { static const struct ib_device_ops ocrdma_dev_ops = { .driver_id = RDMA_DRIVER_OCRDMA, + .uverbs_abi_ver = OCRDMA_ABI_VERSION, .alloc_mr = ocrdma_alloc_mr, .alloc_pd = ocrdma_alloc_pd, @@ -203,7 +204,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, sizeof(OCRDMA_NODE_DESC)); dev->ibdev.owner = THIS_MODULE; - dev->ibdev.uverbs_abi_ver = OCRDMA_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = OCRDMA_UVERBS(GET_CONTEXT) | OCRDMA_UVERBS(QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 737745231f8f..793e25776a7e 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -184,6 +184,7 @@ static void qedr_roce_register_device(struct qedr_dev *dev) static const struct ib_device_ops qedr_dev_ops = { .driver_id = RDMA_DRIVER_QEDR, + .uverbs_abi_ver = QEDR_ABI_VERSION, .alloc_mr = qedr_alloc_mr, .alloc_pd = qedr_alloc_pd, @@ -234,7 +235,6 @@ static int qedr_register_device(struct qedr_dev *dev) dev->ibdev.node_guid = dev->attr.node_guid; memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC)); dev->ibdev.owner = THIS_MODULE; - dev->ibdev.uverbs_abi_ver = QEDR_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) | QEDR_UVERBS(QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index 47830334cb55..f61690816095 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -330,6 +330,7 @@ static void usnic_get_dev_fw_str(struct ib_device *device, char *str) static const struct ib_device_ops usnic_dev_ops = { .driver_id = RDMA_DRIVER_USNIC, + .uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION, .alloc_pd = usnic_ib_alloc_pd, .alloc_ucontext = usnic_ib_alloc_ucontext, @@ -391,7 +392,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev) us_ibdev->ib_dev.phys_port_cnt = USNIC_IB_PORT_CNT; us_ibdev->ib_dev.num_comp_vectors = USNIC_IB_NUM_COMP_VECTORS; us_ibdev->ib_dev.dev.parent = &dev->dev; - us_ibdev->ib_dev.uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION; us_ibdev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 54a0b6372629..2a7eb2838453 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -145,6 +145,7 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num, static const struct ib_device_ops pvrdma_dev_ops = { .driver_id = RDMA_DRIVER_VMW_PVRDMA, + .uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION, .add_gid = pvrdma_add_gid, .alloc_mr = pvrdma_alloc_mr, @@ -203,7 +204,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.num_comp_vectors = 1; dev->ib_dev.dev.parent = &dev->pdev->dev; - dev->ib_dev.uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | -- cgit v1.2.3 From 7a15414252ae4f1d450462d83f883b2d9d8036ee Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Wed, 5 Jun 2019 14:39:26 -0300 Subject: RDMA: Move owner into struct ib_device_ops This more closely follows how other subsytems work, with owner being a member of the structure containing the function pointers. Signed-off-by: Jason Gunthorpe Reviewed-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/bnxt_re/main.c | 2 +- drivers/infiniband/hw/cxgb3/iwch_provider.c | 2 +- drivers/infiniband/hw/cxgb4/provider.c | 2 +- drivers/infiniband/hw/efa/efa_main.c | 2 +- drivers/infiniband/hw/hfi1/verbs.c | 2 +- drivers/infiniband/hw/hns/hns_roce_main.c | 2 +- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2 +- drivers/infiniband/hw/mlx4/main.c | 2 +- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mthca/mthca_provider.c | 3 +-- drivers/infiniband/hw/nes/nes_verbs.c | 2 +- drivers/infiniband/hw/ocrdma/ocrdma_main.c | 2 +- drivers/infiniband/hw/qedr/main.c | 2 +- drivers/infiniband/hw/qib/qib_verbs.c | 2 +- drivers/infiniband/hw/usnic/usnic_ib_main.c | 2 +- drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c | 2 +- 16 files changed, 16 insertions(+), 17 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index a45cb9ee51ab..351c420248a0 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -596,6 +596,7 @@ static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) } static const struct ib_device_ops bnxt_re_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_BNXT_RE, .uverbs_abi_ver = BNXT_RE_ABI_VERSION, @@ -651,7 +652,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) int ret; /* ib device init */ - ibdev->owner = THIS_MODULE; ibdev->node_type = RDMA_NODE_IB_CA; strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", strlen(BNXT_RE_DESC) + 5); diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 5b410605a3a1..1b35941eee74 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -1236,6 +1236,7 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str) } static const struct ib_device_ops iwch_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_CXGB3, .uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION, @@ -1285,7 +1286,6 @@ int iwch_register_device(struct iwch_dev *dev) pr_debug("%s iwch_dev %p\n", __func__, dev); memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6); - dev->ibdev.owner = THIS_MODULE; dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS; diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index c56cdfbd8a88..2b1f2443b7da 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -489,6 +489,7 @@ static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res) } static const struct ib_device_ops c4iw_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_CXGB4, .uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION, @@ -563,7 +564,6 @@ void c4iw_register_device(struct work_struct *work) pr_debug("c4iw_dev %p\n", dev); memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); - dev->ibdev.owner = THIS_MODULE; dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW; if (fastreg_support) dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index b05c5a0b9bc0..b891ee239a67 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -197,6 +197,7 @@ static void efa_stats_init(struct efa_dev *dev) } static const struct ib_device_ops efa_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_EFA, .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION, @@ -262,7 +263,6 @@ static int efa_ib_device_add(struct efa_dev *dev) if (err) goto err_release_doorbell_bar; - dev->ibdev.owner = THIS_MODULE; dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED; dev->ibdev.phys_port_cnt = 1; dev->ibdev.num_comp_vectors = 1; diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index a97f4f9e5c6a..1f36db98240f 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1779,6 +1779,7 @@ static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, } static const struct ib_device_ops hfi1_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_HFI1, .alloc_hw_stats = alloc_hw_stats, @@ -1831,7 +1832,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) */ if (!ib_hfi1_sys_image_guid) ib_hfi1_sys_image_guid = ibdev->node_guid; - ibdev->owner = THIS_MODULE; ibdev->phys_port_cnt = dd->num_pports; ibdev->dev.parent = &dd->pcidev->dev; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index e496b0628e25..f07b2ec86ec2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -414,6 +414,7 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) } static const struct ib_device_ops hns_roce_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_HNS, .uverbs_abi_ver = 1, @@ -483,7 +484,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_dev = &hr_dev->ib_dev; - ib_dev->owner = THIS_MODULE; ib_dev->node_type = RDMA_NODE_IB_CA; ib_dev->dev.parent = dev; diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 4dc647c8556b..bfe16e6f04f4 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -2650,6 +2650,7 @@ static int i40iw_query_pkey(struct ib_device *ibdev, } static const struct ib_device_ops i40iw_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_I40IW, /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ .uverbs_abi_ver = I40IW_ABI_VER, @@ -2711,7 +2712,6 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev i40iw_pr_err("iwdev == NULL\n"); return NULL; } - iwibdev->ibdev.owner = THIS_MODULE; iwdev->iwibdev = iwibdev; iwibdev->iwdev = iwdev; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 1f87221acbb0..5d7a87842291 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -2510,6 +2510,7 @@ static void get_fw_ver_str(struct ib_device *device, char *str) } static const struct ib_device_ops mlx4_ib_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_MLX4, .uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION, @@ -2646,7 +2647,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ibdev->dev = dev; ibdev->bond_next_port = 0; - ibdev->ib_dev.owner = THIS_MODULE; ibdev->ib_dev.node_type = RDMA_NODE_IB_CA; ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey; ibdev->num_ports = num_ports; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0c23eccb8855..1e3d936ed809 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6044,7 +6044,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) if (mlx5_use_mad_ifc(dev)) get_ext_port_caps(dev); - dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.node_type = RDMA_NODE_IB_CA; dev->ib_dev.local_dma_lkey = 0 /* not supported for now */; dev->ib_dev.phys_port_cnt = dev->num_ports; @@ -6124,6 +6123,7 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) } static const struct ib_device_ops mlx5_ib_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_MLX5, .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION, diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 690c65accea4..b128ff76f709 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -1153,6 +1153,7 @@ static void get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops mthca_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_MTHCA, .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, @@ -1246,8 +1247,6 @@ int mthca_register_device(struct mthca_dev *dev) if (ret) return ret; - dev->ib_dev.owner = THIS_MODULE; - dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index f1fdd6829a40..db044b2eaead 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -3558,6 +3558,7 @@ static void get_dev_fw_str(struct ib_device *dev, char *str) } static const struct ib_device_ops nes_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_NES, /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ .uverbs_abi_ver = NES_ABI_USERSPACE_VER, @@ -3617,7 +3618,6 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) if (nesibdev == NULL) { return NULL; } - nesibdev->ibdev.owner = THIS_MODULE; nesibdev->ibdev.node_type = RDMA_NODE_RNIC; memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid)); diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index ef823f1144b5..b326313d413f 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -144,6 +144,7 @@ static const struct attribute_group ocrdma_attr_group = { }; static const struct ib_device_ops ocrdma_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_OCRDMA, .uverbs_abi_ver = OCRDMA_ABI_VERSION, @@ -203,7 +204,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX); memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, sizeof(OCRDMA_NODE_DESC)); - dev->ibdev.owner = THIS_MODULE; dev->ibdev.uverbs_cmd_mask = OCRDMA_UVERBS(GET_CONTEXT) | OCRDMA_UVERBS(QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 793e25776a7e..a0bb07ba0f3c 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -183,6 +183,7 @@ static void qedr_roce_register_device(struct qedr_dev *dev) } static const struct ib_device_ops qedr_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_QEDR, .uverbs_abi_ver = QEDR_ABI_VERSION, @@ -234,7 +235,6 @@ static int qedr_register_device(struct qedr_dev *dev) dev->ibdev.node_guid = dev->attr.node_guid; memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC)); - dev->ibdev.owner = THIS_MODULE; dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) | QEDR_UVERBS(QUERY_DEVICE) | diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index bbc331d7f49b..54310fd6c7b6 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -1482,6 +1482,7 @@ static void qib_fill_device_attr(struct qib_devdata *dd) } static const struct ib_device_ops qib_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_QIB, .init_port = qib_create_port_files, @@ -1547,7 +1548,6 @@ int qib_register_ib_device(struct qib_devdata *dd) if (!ib_qib_sys_image_guid) ib_qib_sys_image_guid = ppd->guid; - ibdev->owner = THIS_MODULE; ibdev->node_guid = ppd->guid; ibdev->phys_port_cnt = dd->num_pports; ibdev->dev.parent = &dd->pcidev->dev; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index f61690816095..e701322dc740 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -329,6 +329,7 @@ static void usnic_get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops usnic_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_USNIC, .uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION, @@ -387,7 +388,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev) us_ibdev->pdev = dev; us_ibdev->netdev = pci_get_drvdata(dev); - us_ibdev->ib_dev.owner = THIS_MODULE; us_ibdev->ib_dev.node_type = RDMA_NODE_USNIC_UDP; us_ibdev->ib_dev.phys_port_cnt = USNIC_IB_PORT_CNT; us_ibdev->ib_dev.num_comp_vectors = USNIC_IB_NUM_COMP_VECTORS; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 2a7eb2838453..0c48464ffff1 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -144,6 +144,7 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num, } static const struct ib_device_ops pvrdma_dev_ops = { + .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_VMW_PVRDMA, .uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION, @@ -201,7 +202,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) dev->ib_dev.node_guid = dev->dsr->caps.node_guid; dev->sys_image_guid = dev->dsr->caps.sys_image_guid; dev->flags = 0; - dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.num_comp_vectors = 1; dev->ib_dev.dev.parent = &dev->pdev->dev; dev->ib_dev.uverbs_cmd_mask = -- cgit v1.2.3 From 147b308e6a63a0572b750b1dc9b8b6fc33997e4b Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 28 May 2019 14:37:27 +0300 Subject: RDMA/nes: Avoid memory allocation during CQ destroy The memory allocation call can fail and cause to early return from nes_desotroy_cq() function. This situation will cause to memory leak of struct nes_cq. Rewrite function to avoid memory allocation. Signed-off-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/nes/nes_verbs.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index db044b2eaead..7677f1f734bb 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -1641,7 +1641,7 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct nes_vnic *nesvnic; struct nes_adapter *nesadapter; struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; + struct nes_cqp_request cqp_request = {}; unsigned long flags; u32 opcode = 0; int ret; @@ -1654,13 +1654,10 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number); /* Send DestroyCQ request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; + INIT_LIST_HEAD(&cqp_request.list); + init_waitqueue_head(&cqp_request.waitq); + cqp_request.waiting = 1; + cqp_wqe = &cqp_request.cqp_wqe; opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); spin_lock_irqsave(&nesadapter->pbl_lock, flags); if (nescq->virtual_cq == 1) { @@ -1687,30 +1684,28 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (!nescq->mcrqf) nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number); - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); + nes_post_cqp_request(nesdev, &cqp_request); /* Wait for CQP */ nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", nescq->hw_cq.cq_number); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); + ret = wait_event_timeout(cqp_request.waitq, cqp_request.request_done, + NES_EVENT_TIMEOUT); nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u," " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nescq->hw_cq.cq_number, ret, cqp_request->major_code, - cqp_request->minor_code); + nescq->hw_cq.cq_number, ret, cqp_request.major_code, + cqp_request.minor_code); if (!ret) { nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n", nescq->hw_cq.cq_number); ret = -ETIME; - } else if (cqp_request->major_code) { + } else if (cqp_request.major_code) { nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n", nescq->hw_cq.cq_number); ret = -EIO; } else { ret = 0; } - nes_put_cqp_request(nesdev, cqp_request); if (nescq->cq_mem_size) pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, @@ -1899,7 +1894,6 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, } barrier(); - atomic_set(&cqp_request->refcount, 2); nes_post_cqp_request(nesdev, cqp_request); /* Wait for CQP */ -- cgit v1.2.3 From a52c8e2469c30cf7ac453d624aed9c168b23d1af Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 28 May 2019 14:37:28 +0300 Subject: RDMA: Clean destroy CQ in drivers do not return errors Like all other destroy commands, .destroy_cq() call is not supposed to fail. In all flows, the attempt to return earlier caused to memory leaks. This patch converts .destroy_cq() to do not return any errors. Signed-off-by: Leon Romanovsky Acked-by: Gal Pressman Acked-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 13 ++----- drivers/infiniband/hw/bnxt_re/ib_verbs.h | 2 +- drivers/infiniband/hw/cxgb3/cxio_hal.c | 6 ++-- drivers/infiniband/hw/cxgb3/cxio_hal.h | 2 +- drivers/infiniband/hw/cxgb3/iwch_provider.c | 3 +- drivers/infiniband/hw/cxgb4/cq.c | 13 +++---- drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 2 +- drivers/infiniband/hw/efa/efa.h | 2 +- drivers/infiniband/hw/efa/efa_verbs.c | 9 ++--- drivers/infiniband/hw/hns/hns_roce_cq.c | 48 ++++++++++++------------- drivers/infiniband/hw/hns/hns_roce_device.h | 4 +-- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 14 ++------ drivers/infiniband/hw/i40iw/i40iw_verbs.c | 3 +- drivers/infiniband/hw/mlx4/cq.c | 4 +-- drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +- drivers/infiniband/hw/mlx5/cq.c | 4 +-- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- drivers/infiniband/hw/mthca/mthca_provider.c | 4 +-- drivers/infiniband/hw/nes/nes_utils.c | 4 +-- drivers/infiniband/hw/nes/nes_verbs.c | 30 +++++----------- drivers/infiniband/hw/ocrdma/ocrdma_hw.c | 8 ++--- drivers/infiniband/hw/ocrdma/ocrdma_hw.h | 2 +- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 6 ++-- drivers/infiniband/hw/ocrdma/ocrdma_verbs.h | 2 +- drivers/infiniband/hw/qedr/verbs.c | 20 ++--------- drivers/infiniband/hw/qedr/verbs.h | 2 +- drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 4 +-- drivers/infiniband/hw/usnic/usnic_ib_verbs.h | 2 +- drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c | 6 +--- drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h | 2 +- 30 files changed, 74 insertions(+), 151 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 8af8e1472101..0127af45dcd1 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2517,9 +2517,8 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, } /* Completion Queues */ -int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { - int rc; struct bnxt_re_cq *cq; struct bnxt_qplib_nq *nq; struct bnxt_re_dev *rdev; @@ -2528,20 +2527,14 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) rdev = cq->rdev; nq = cq->qplib_cq.nq; - rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); - if (rc) { - dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); - return rc; - } - if (!IS_ERR_OR_NULL(cq->umem)) + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (!cq->umem) ib_umem_release(cq->umem); atomic_dec(&rdev->cq_count); nq->budget--; kfree(cq->cql); kfree(cq); - - return 0; } struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 09a33049e42f..828403ee0104 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -193,7 +193,7 @@ int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +void bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index d9c741fea0e9..37ee93824349 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -303,17 +303,15 @@ err1: return -ENOMEM; } -int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) +void cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) { - int err; - err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid); + cxio_hal_clear_cq_ctx(rdev_p, cq->cqid); kfree(cq->sw_queue); dma_free_coherent(&(rdev_p->rnic_info.pdev->dev), (1UL << (cq->size_log2)) * sizeof(struct t3_cqe) + 1, cq->queue, dma_unmap_addr(cq, mapping)); cxio_hal_put_cqid(rdev_p->rscp, cq->cqid); - return err; } int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq, diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h index 5fc26a4648d3..40c029ffa425 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.h +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h @@ -158,7 +158,7 @@ void cxio_rdev_close(struct cxio_rdev *rdev); int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq, enum t3_cq_opcode op, u32 credit); int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq, int kernel); -int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq); +void cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq); void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq, diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 1b35941eee74..5bde4ae93681 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -88,7 +88,7 @@ static int iwch_alloc_ucontext(struct ib_ucontext *ucontext, return 0; } -static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static void iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct iwch_cq *chp; @@ -101,7 +101,6 @@ static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); kfree(chp); - return 0; } static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 6557e7c5af66..f49e6d271c42 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -34,14 +34,13 @@ #include "iw_cxgb4.h" -static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, - struct c4iw_dev_ucontext *uctx, struct sk_buff *skb, - struct c4iw_wr_wait *wr_waitp) +static void destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, + struct c4iw_dev_ucontext *uctx, struct sk_buff *skb, + struct c4iw_wr_wait *wr_waitp) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; - int ret; wr_len = sizeof(*res_wr) + sizeof(*res); set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); @@ -59,14 +58,13 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(wr_waitp); - ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); + c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); - return ret; } static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, @@ -970,7 +968,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) return !err || err == -ENODATA ? npolled : err; } -int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct c4iw_cq *chp; struct c4iw_ucontext *ucontext; @@ -989,7 +987,6 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) chp->destroy_skb, chp->wr_waitp); c4iw_put_wr_wait(chp->wr_waitp); kfree(chp); - return 0; } struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index cf7512b2c4c0..45e720288f0f 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -992,7 +992,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, struct ib_udata *udata); struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc); int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); -int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 14a36546985b..52d894f0ad3e 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -134,7 +134,7 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); struct ib_qp *efa_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); -int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); struct ib_cq *efa_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 607aff869200..42865cf4f149 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -847,25 +847,20 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) return efa_com_destroy_cq(&dev->edev, ¶ms); } -int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct efa_dev *dev = to_edev(ibcq->device); struct efa_cq *cq = to_ecq(ibcq); - int err; ibdev_dbg(&dev->ibdev, "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n", cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr); - err = efa_destroy_cq_idx(dev, cq->cq_idx); - if (err) - return err; - + efa_destroy_cq_idx(dev, cq->cq_idx); dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size, DMA_FROM_DEVICE); kfree(cq); - return 0; } static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 6e81ff3f1813..0eb7c16c007b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -443,40 +443,36 @@ err_cq: } EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq); -int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); - int ret = 0; if (hr_dev->hw->destroy_cq) { - ret = hr_dev->hw->destroy_cq(ib_cq, udata); - } else { - hns_roce_free_cq(hr_dev, hr_cq); - hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - - if (udata) { - ib_umem_release(hr_cq->umem); - - if (hr_cq->db_en == 1) - hns_roce_db_unmap_user( - rdma_udata_to_drv_context( - udata, - struct hns_roce_ucontext, - ibucontext), - &hr_cq->db); - } else { - /* Free the buff of stored cq */ - hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, - ib_cq->cqe); - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) - hns_roce_free_db(hr_dev, &hr_cq->db); - } + hr_dev->hw->destroy_cq(ib_cq, udata); + return; + } + + hns_roce_free_cq(hr_dev, hr_cq); + hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - kfree(hr_cq); + if (udata) { + ib_umem_release(hr_cq->umem); + + if (hr_cq->db_en == 1) + hns_roce_db_unmap_user(rdma_udata_to_drv_context( + udata, + struct hns_roce_ucontext, + ibucontext), + &hr_cq->db); + } else { + /* Free the buff of stored cq */ + hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) + hns_roce_free_db(hr_dev, &hr_cq->db); } - return ret; + kfree(hr_cq); } EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index ce23338831eb..2f7d9644fd24 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -938,7 +938,7 @@ struct hns_roce_hw { int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, struct ib_udata *udata); - int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); + void (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); @@ -1209,7 +1209,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq); int hns_roce_db_map_user(struct hns_roce_ucontext *context, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 878c8ae35630..aa7b67d283af 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -865,8 +865,7 @@ alloc_pd_failed: kfree(pd); alloc_mem_failed: - if (hns_roce_ib_destroy_cq(cq, NULL)) - dev_err(dev, "Destroy cq for create_lp_qp failed!\n"); + hns_roce_ib_destroy_cq(cq, NULL); return ret; } @@ -894,10 +893,7 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) i, ret); } - ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); - if (ret) - dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret); - + hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL); } @@ -3654,7 +3650,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) return 0; } -static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); @@ -3663,7 +3659,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) u32 cqe_cnt_cur; u32 cq_buf_size; int wait_time = 0; - int ret = 0; hns_roce_free_cq(hr_dev, hr_cq); @@ -3685,7 +3680,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) { dev_warn(dev, "Destroy cq 0x%lx timeout!\n", hr_cq->cqn); - ret = -ETIMEDOUT; break; } wait_time++; @@ -3702,8 +3696,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) } kfree(hr_cq); - - return ret; } static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index bfe16e6f04f4..205053cb5f97 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -1064,7 +1064,7 @@ void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq) * @ib_cq: cq pointer * @udata: user data or NULL for kernel object */ -static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct i40iw_cq *iwcq; struct i40iw_device *iwdev; @@ -1077,7 +1077,6 @@ static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) cq_free_resources(iwdev, iwcq); kfree(iwcq); i40iw_rem_devusecount(iwdev); - return 0; } /** diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 022a0b4ea452..8eb7490dabb8 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -486,7 +486,7 @@ out: return err; } -int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(cq->device); struct mlx4_ib_cq *mcq = to_mcq(cq); @@ -508,8 +508,6 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) } kfree(mcq); - - return 0; } static void dump_cqe(void *cqe) diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 26897102057d..af5ee45a9f19 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -746,7 +746,7 @@ int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 2e2e65f00257..ebd01bd7f8f6 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -998,7 +998,7 @@ err_create: return ERR_PTR(err); } -int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(cq->device); struct mlx5_ib_cq *mcq = to_mcq(cq); @@ -1010,8 +1010,6 @@ int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) destroy_cq_kernel(dev, mcq); kfree(mcq); - - return 0; } static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 40eb8be482e4..bf697c5b3e79 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1118,7 +1118,7 @@ int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index b128ff76f709..81fc04e1c142 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -804,7 +804,7 @@ out: return ret; } -static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { if (udata) { struct mthca_ucontext *context = @@ -824,8 +824,6 @@ static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) } mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); kfree(cq); - - return 0; } static inline u32 convert_access(int acc) diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c index 90f28890246d..e976292fc6c0 100644 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ b/drivers/infiniband/hw/nes/nes_utils.c @@ -588,9 +588,7 @@ struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev) cqp_request->callback = 0; nes_debug(NES_DBG_CQP, "Got cqp request %p from the available list \n", cqp_request); - } else - printk(KERN_ERR PFX "%s: Could not allocated a CQP request.\n", - __func__); + } return cqp_request; } diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 7677f1f734bb..cac3fa624c4d 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -1634,7 +1634,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, /** * nes_destroy_cq */ -static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static void nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct nes_cq *nescq; struct nes_device *nesdev; @@ -1644,7 +1644,6 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) struct nes_cqp_request cqp_request = {}; unsigned long flags; u32 opcode = 0; - int ret; nescq = to_nescq(ib_cq); nesvnic = to_nesvnic(ib_cq->device); @@ -1656,6 +1655,7 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) /* Send DestroyCQ request to CQP */ INIT_LIST_HEAD(&cqp_request.list); init_waitqueue_head(&cqp_request.waitq); + cqp_request.waiting = 1; cqp_wqe = &cqp_request.cqp_wqe; opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); @@ -1689,30 +1689,18 @@ static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) /* Wait for CQP */ nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", nescq->hw_cq.cq_number); - ret = wait_event_timeout(cqp_request.waitq, cqp_request.request_done, - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nescq->hw_cq.cq_number, ret, cqp_request.major_code, - cqp_request.minor_code); - if (!ret) { - nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n", - nescq->hw_cq.cq_number); - ret = -ETIME; - } else if (cqp_request.major_code) { - nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n", - nescq->hw_cq.cq_number); - ret = -EIO; - } else { - ret = 0; - } + wait_event_timeout(cqp_request.waitq, cqp_request.request_done, + NES_EVENT_TIMEOUT); + nes_debug( + NES_DBG_CQ, + "Destroy iWARP CQ%u completed CQP Major:Minor codes = 0x%04X:0x%04X.\n", + nescq->hw_cq.cq_number, cqp_request.major_code, + cqp_request.minor_code); if (nescq->cq_mem_size) pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); kfree(nescq); - - return ret; } /** diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index 5127e2ea4bdd..b2dd4e0a4be2 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1888,14 +1888,13 @@ mem_err: return status; } -int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) +void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) { - int status = -ENOMEM; struct ocrdma_destroy_cq *cmd; cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_CQ, sizeof(*cmd)); if (!cmd) - return status; + return; ocrdma_init_mch(&cmd->req, OCRDMA_CMD_DELETE_CQ, OCRDMA_SUBSYS_COMMON, sizeof(*cmd)); @@ -1903,11 +1902,10 @@ int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) (cq->id << OCRDMA_DESTROY_CQ_QID_SHIFT) & OCRDMA_DESTROY_CQ_QID_MASK; - status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); + ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); ocrdma_unbind_eq(dev, cq->eqn); dma_free_coherent(&dev->nic_info.pdev->dev, cq->len, cq->va, cq->pa); kfree(cmd); - return status; } int ocrdma_mbx_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h index 06ec59326a90..12c23a7652b9 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h @@ -122,7 +122,7 @@ int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr, u32 pd_id, int acc); int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *, int entries, int dpp_cq, u16 pd_id); -int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *); +void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq); int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs, u8 enable_dpp_cq, u16 dpp_cq_id, u16 *dpp_offset, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 35ec87015792..94e4f7f9b1f7 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -1070,7 +1070,7 @@ static void ocrdma_flush_cq(struct ocrdma_cq *cq) spin_unlock_irqrestore(&cq->cq_lock, flags); } -int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); struct ocrdma_eq *eq = NULL; @@ -1080,14 +1080,13 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) dev->cq_tbl[cq->id] = NULL; indx = ocrdma_get_eq_table_index(dev, cq->eqn); - BUG_ON(indx == -EINVAL); eq = &dev->eq_tbl[indx]; irq = ocrdma_get_irq(dev, eq); synchronize_irq(irq); ocrdma_flush_cq(cq); - (void)ocrdma_mbx_destroy_cq(dev, cq); + ocrdma_mbx_destroy_cq(dev, cq); if (cq->ucontext) { pdid = cq->ucontext->cntxt_pd->id; ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, @@ -1098,7 +1097,6 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) } kfree(cq); - return 0; } static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index d76aae7ed0d3..89cebe05669e 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -75,7 +75,7 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); struct ib_qp *ocrdma_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 3c0dba072071..be29bbbc4b14 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -955,14 +955,13 @@ int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) #define QEDR_DESTROY_CQ_ITER_DURATION (10) -int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct qedr_dev *dev = get_qedr_dev(ibcq->device); struct qed_rdma_destroy_cq_out_params oparams; struct qed_rdma_destroy_cq_in_params iparams; struct qedr_cq *cq = get_qedr_cq(ibcq); int iter; - int rc; DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); @@ -973,10 +972,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) goto done; iparams.icid = cq->icid; - rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); - if (rc) - return rc; - + dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); dev->ops->common->chain_free(dev->cdev, &cq->pbl); if (udata) { @@ -1007,9 +1003,6 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) iter--; } - if (oparams.num_cq_notif != cq->cnq_notif) - goto err; - /* Note that we don't need to have explicit code to wait for the * completion of the event handler because it is invoked from the EQ. * Since the destroy CQ ramrod has also been received on the EQ we can @@ -1019,15 +1012,6 @@ done: cq->sig = ~cq->sig; kfree(cq); - - return 0; - -err: - DP_ERR(dev, - "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", - cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); - - return -EINVAL; } static inline int get_gid_info_from_table(struct ib_qp *ibqp, diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 9328c80375ef..32d7ce77e339 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -54,7 +54,7 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, struct ib_udata *); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index e9352750e029..5686d14b86fe 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -604,11 +604,9 @@ struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, return cq; } -int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { - usnic_dbg("\n"); kfree(cq); - return 0; } struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 028f322f8e9b..0b9d993433a7 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -61,7 +61,7 @@ int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index d7deb19a2800..0682781f6555 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -246,10 +246,8 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) * pvrdma_destroy_cq - destroy completion queue * @cq: the completion queue to destroy. * @udata: user data or null for kernel object - * - * @return: 0 for success. */ -int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct pvrdma_cq *vcq = to_vcq(cq); union pvrdma_cmd_req req; @@ -275,8 +273,6 @@ int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) pvrdma_free_cq(dev, vcq); atomic_dec(&dev->num_cqs); - - return ret; } static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i) diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index 9d7b021e1c59..f0dd6e4d058b 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -412,7 +412,7 @@ int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr, struct ib_udata *udata); -int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags, -- cgit v1.2.3 From e39afe3d6dbd908d8fd189571a3c1561088a86c2 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Tue, 28 May 2019 14:37:29 +0300 Subject: RDMA: Convert CQ allocations to be under core responsibility Ensure that CQ is allocated and freed by IB/core and not by drivers. Signed-off-by: Leon Romanovsky Acked-by: Gal Pressman Reviewed-by: Dennis Dalessandro Tested-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 20 +++------ drivers/infiniband/hw/bnxt_re/ib_verbs.h | 7 ++- drivers/infiniband/hw/bnxt_re/main.c | 1 + drivers/infiniband/hw/cxgb3/iwch_provider.c | 43 ++++++++---------- drivers/infiniband/hw/cxgb4/cq.c | 27 +++++------ drivers/infiniband/hw/cxgb4/iw_cxgb4.h | 5 +-- drivers/infiniband/hw/cxgb4/provider.c | 1 + drivers/infiniband/hw/efa/efa.h | 5 +-- drivers/infiniband/hw/efa/efa_main.c | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 48 ++++++-------------- drivers/infiniband/hw/hns/hns_roce_cq.c | 23 ++++------ drivers/infiniband/hw/hns/hns_roce_device.h | 6 +-- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 21 +++++---- drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/i40iw/i40iw_verbs.c | 33 ++++++-------- drivers/infiniband/hw/mlx4/cq.c | 25 ++++------- drivers/infiniband/hw/mlx4/main.c | 1 + drivers/infiniband/hw/mlx4/mlx4_ib.h | 5 +-- drivers/infiniband/hw/mlx5/cq.c | 32 +++++-------- drivers/infiniband/hw/mlx5/main.c | 21 +++++---- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +-- drivers/infiniband/hw/mthca/mthca_provider.c | 36 +++++++-------- drivers/infiniband/hw/nes/nes_verbs.c | 60 +++++++++---------------- drivers/infiniband/hw/ocrdma/ocrdma_main.c | 1 + drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 29 +++++------- drivers/infiniband/hw/ocrdma/ocrdma_verbs.h | 5 +-- drivers/infiniband/hw/qedr/main.c | 1 + drivers/infiniband/hw/qedr/verbs.c | 28 ++++-------- drivers/infiniband/hw/qedr/verbs.h | 5 +-- drivers/infiniband/hw/usnic/usnic_ib.h | 4 ++ drivers/infiniband/hw/usnic/usnic_ib_main.c | 1 + drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 18 +++----- drivers/infiniband/hw/usnic/usnic_ib_verbs.h | 5 +-- drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c | 34 +++++--------- drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c | 1 + drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h | 5 +-- 36 files changed, 223 insertions(+), 341 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 0127af45dcd1..44cc5f19df3b 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2534,16 +2534,14 @@ void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) atomic_dec(&rdev->cq_count); nq->budget--; kfree(cq->cql); - kfree(cq); } -struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; - struct bnxt_re_cq *cq = NULL; + struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); int rc, entries; int cqe = attr->cqe; struct bnxt_qplib_nq *nq = NULL; @@ -2552,11 +2550,8 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, /* Validate CQ fields */ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); - return ERR_PTR(-EINVAL); + return -EINVAL; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); cq->rdev = rdev; cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); @@ -2634,15 +2629,14 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, } } - return &cq->ib_cq; + return 0; c2fail: if (udata) ib_umem_release(cq->umem); fail: kfree(cq->cql); - kfree(cq); - return ERR_PTR(rc); + return rc; } static u8 __req_to_ib_wc_status(u8 qstatus) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 828403ee0104..31662b1ee35a 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -94,11 +94,11 @@ struct bnxt_re_qp { }; struct bnxt_re_cq { + struct ib_cq ib_cq; struct bnxt_re_dev *rdev; spinlock_t cq_lock; /* protect cq */ u16 cq_count; u16 cq_period; - struct ib_cq ib_cq; struct bnxt_qplib_cq qplib_cq; struct bnxt_qplib_cqe *cql; #define MAX_CQL_PER_POLL 1024 @@ -190,9 +190,8 @@ int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr, const struct ib_send_wr **bad_send_wr); int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr **bad_recv_wr); -struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 351c420248a0..029babe713f3 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -641,6 +641,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { .reg_user_mr = bnxt_re_reg_user_mr, .req_notify_cq = bnxt_re_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah), + INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd), INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq), INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx), diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 5bde4ae93681..acba96f289cc 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -100,16 +100,16 @@ static void iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) wait_event(chp->wait, !atomic_read(&chp->refcnt)); cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); - kfree(chp); } -static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int iwch_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; - struct iwch_dev *rhp; - struct iwch_cq *chp; + struct iwch_dev *rhp = to_iwch_dev(ibcq->device); + struct iwch_cq *chp = to_iwch_cq(ibcq); struct iwch_create_cq_resp uresp; struct iwch_create_cq_req ureq; static int warned; @@ -117,19 +117,13 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries); if (attr->flags) - return ERR_PTR(-EINVAL); - - rhp = to_iwch_dev(ibdev); - chp = kzalloc(sizeof(*chp), GFP_KERNEL); - if (!chp) - return ERR_PTR(-ENOMEM); + return -EINVAL; if (udata) { if (!t3a_device(rhp)) { - if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { - kfree(chp); - return ERR_PTR(-EFAULT); - } + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) + return -EFAULT; + chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr; } } @@ -150,10 +144,9 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, entries = roundup_pow_of_two(entries); chp->cq.size_log2 = ilog2(entries); - if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) { - kfree(chp); - return ERR_PTR(-ENOMEM); - } + if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) + return -ENOMEM; + chp->rhp = rhp; chp->ibcq.cqe = 1 << chp->cq.size_log2; spin_lock_init(&chp->lock); @@ -162,8 +155,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, init_waitqueue_head(&chp->wait); if (xa_store_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL)) { cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); - kfree(chp); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } if (udata) { @@ -174,7 +166,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, mm = kmalloc(sizeof(*mm), GFP_KERNEL); if (!mm) { iwch_destroy_cq(&chp->ibcq, udata); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } uresp.cqid = chp->cq.cqid; uresp.size_log2 = chp->cq.size_log2; @@ -200,14 +192,14 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, if (ib_copy_to_udata(udata, &uresp, resplen)) { kfree(mm); iwch_destroy_cq(&chp->ibcq, udata); - return ERR_PTR(-EFAULT); + return -EFAULT; } insert_mmap(ucontext, mm); } pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr %pad\n", chp->cq.cqid, chp, (1 << chp->cq.size_log2), &chp->cq.dma_addr); - return &chp->ibcq; + return 0; } static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) @@ -1277,6 +1269,7 @@ static const struct ib_device_ops iwch_dev_ops = { .reg_user_mr = iwch_reg_user_mr, .req_notify_cq = iwch_arm_cq, INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, iwch_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index f49e6d271c42..3cc4d3331a3f 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -986,17 +986,16 @@ void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx, chp->destroy_skb, chp->wr_waitp); c4iw_put_wr_wait(chp->wr_waitp); - kfree(chp); } -struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; - struct c4iw_dev *rhp; - struct c4iw_cq *chp; + struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device); + struct c4iw_cq *chp = to_c4iw_cq(ibcq); struct c4iw_create_cq ucmd; struct c4iw_create_cq_resp uresp; int ret, wr_len; @@ -1007,22 +1006,16 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, pr_debug("ib_dev %p entries %d\n", ibdev, entries); if (attr->flags) - return ERR_PTR(-EINVAL); - - rhp = to_c4iw_dev(ibdev); + return -EINVAL; if (vector >= rhp->rdev.lldi.nciq) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { if (udata->inlen < sizeof(ucmd)) ucontext->is_32b_cqe = 1; } - chp = kzalloc(sizeof(*chp), GFP_KERNEL); - if (!chp) - return ERR_PTR(-ENOMEM); - chp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL); if (!chp->wr_waitp) { ret = -ENOMEM; @@ -1132,10 +1125,11 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, mm2->len = PAGE_SIZE; insert_mmap(ucontext, mm2); } + pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr %pad\n", chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize, &chp->cq.dma_addr); - return &chp->ibcq; + return 0; err_free_mm2: kfree(mm2); err_free_mm: @@ -1151,8 +1145,7 @@ err_free_skb: err_free_wr_wait: c4iw_put_wr_wait(chp->wr_waitp); err_free_chp: - kfree(chp); - return ERR_PTR(ret); + return ret; } int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 45e720288f0f..7d06b0f8d49a 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -993,9 +993,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc); int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); -struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr, enum ib_srq_attr_mask srq_attr_mask, diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 2b1f2443b7da..5e59c5708729 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -537,6 +537,7 @@ static const struct ib_device_ops c4iw_dev_ops = { .reg_user_mr = c4iw_reg_user_mr, .req_notify_cq = c4iw_arm_cq, INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 52d894f0ad3e..119f8efec564 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -135,9 +135,8 @@ struct ib_qp *efa_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); -struct ib_cq *efa_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index b891ee239a67..46861461dd2d 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -224,6 +224,7 @@ static const struct ib_device_ops efa_dev_ops = { .reg_user_mr = efa_reg_mr, INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 42865cf4f149..a9372c9e4b30 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -859,8 +859,6 @@ void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) efa_destroy_cq_idx(dev, cq->cq_idx); dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size, DMA_FROM_DEVICE); - - kfree(cq); } static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, @@ -876,17 +874,20 @@ static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, return 0; } -static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, - int vector, struct ib_ucontext *ibucontext, - struct ib_udata *udata) +int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct efa_ucontext *ucontext = rdma_udata_to_drv_context( + udata, struct efa_ucontext, ibucontext); struct efa_ibv_create_cq_resp resp = {}; struct efa_com_create_cq_params params; struct efa_com_create_cq_result result; + struct ib_device *ibdev = ibcq->device; struct efa_dev *dev = to_edev(ibdev); struct efa_ibv_create_cq cmd = {}; + struct efa_cq *cq = to_ecq(ibcq); bool cq_entry_inserted = false; - struct efa_cq *cq; + int entries = attr->cqe; int err; ibdev_dbg(ibdev, "create_cq entries %d\n", entries); @@ -944,19 +945,13 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, goto err_out; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - err = -ENOMEM; - goto err_out; - } - - cq->ucontext = to_eucontext(ibucontext); + cq->ucontext = ucontext; cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, DMA_FROM_DEVICE); if (!cq->cpu_addr) { err = -ENOMEM; - goto err_free_cq; + goto err_out; } params.uarn = cq->ucontext->uarn; @@ -975,8 +970,8 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, err = cq_mmap_entries_setup(dev, cq, &resp); if (err) { - ibdev_dbg(ibdev, - "Could not setup cq[%u] mmap entries\n", cq->cq_idx); + ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n", + cq->cq_idx); goto err_destroy_cq; } @@ -992,11 +987,10 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, } } - ibdev_dbg(ibdev, - "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", + ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr); - return &cq->ibcq; + return 0; err_destroy_cq: efa_destroy_cq_idx(dev, cq->cq_idx); @@ -1005,23 +999,9 @@ err_free_mapped: DMA_FROM_DEVICE); if (!cq_entry_inserted) free_pages_exact(cq->cpu_addr, cq->size); -err_free_cq: - kfree(cq); err_out: atomic64_inc(&dev->stats.sw_stats.create_cq_err); - return ERR_PTR(err); -} - -struct ib_cq *efa_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -{ - struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata, - struct efa_ucontext, - ibucontext); - - return do_create_cq(ibdev, attr->cqe, attr->comp_vector, - &ucontext->ibucontext, udata); + return err; } static int umem_to_page_list(struct efa_dev *dev, diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 0eb7c16c007b..7e198c9ffbfe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -299,15 +299,15 @@ static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev, &buf->hr_buf); } -struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int hns_roce_ib_create_cq(struct ib_cq *ib_cq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct device *dev = hr_dev->dev; struct hns_roce_ib_create_cq ucmd; struct hns_roce_ib_create_cq_resp resp = {}; - struct hns_roce_cq *hr_cq = NULL; + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); struct hns_roce_uar *uar = NULL; int vector = attr->comp_vector; int cq_entries = attr->cqe; @@ -318,13 +318,9 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n", cq_entries, hr_dev->caps.max_cqes); - return ERR_PTR(-EINVAL); + return -EINVAL; } - hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); - if (!hr_cq) - return ERR_PTR(-ENOMEM); - if (hr_dev->caps.min_cqes) cq_entries = max(cq_entries, hr_dev->caps.min_cqes); @@ -415,7 +411,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, goto err_cqc; } - return &hr_cq->ib_cq; + return 0; err_cqc: hns_roce_free_cq(hr_dev, hr_cq); @@ -438,8 +434,7 @@ err_db: hns_roce_free_db(hr_dev, &hr_cq->db); err_cq: - kfree(hr_cq); - return ERR_PTR(ret); + return ret; } EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq); @@ -471,8 +466,6 @@ void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) hns_roce_free_db(hr_dev, &hr_cq->db); } - - kfree(hr_cq); } EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 2f7d9644fd24..303ea7c614a8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1205,9 +1205,9 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, __be32 send_ieth(const struct ib_send_wr *wr); int to_hr_qp_type(int qp_type); -struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int hns_roce_ib_create_cq(struct ib_cq *ib_cq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index aa7b67d283af..c899879da222 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -717,7 +717,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) union ib_gid dgid; u64 subnet_prefix; int attr_mask = 0; - int ret = -ENOMEM; + int ret; int i, j; u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 }; u8 phy_port; @@ -730,10 +730,16 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) /* Reserved cq for loop qp */ cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2; cq_init_attr.comp_vector = 0; - cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL); - if (IS_ERR(cq)) { - dev_err(dev, "Create cq for reserved loop qp failed!"); + + ibdev = &hr_dev->ib_dev; + cq = rdma_zalloc_drv_obj(ibdev, ib_cq); + if (!cq) return -ENOMEM; + + ret = hns_roce_ib_create_cq(cq, &cq_init_attr, NULL); + if (ret) { + dev_err(dev, "Create cq for reserved loop qp failed!"); + goto alloc_cq_failed; } free_mr->mr_free_cq = to_hr_cq(cq); free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev; @@ -743,7 +749,6 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) free_mr->mr_free_cq->ib_cq.cq_context = NULL; atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0); - ibdev = &hr_dev->ib_dev; pd = rdma_zalloc_drv_obj(ibdev, ib_pd); if (!pd) goto alloc_mem_failed; @@ -866,7 +871,8 @@ alloc_pd_failed: alloc_mem_failed: hns_roce_ib_destroy_cq(cq, NULL); - +alloc_cq_failed: + kfree(cq); return ret; } @@ -894,6 +900,7 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) } hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); + kfree(&free_mr->mr_free_cq->ib_cq); hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL); } @@ -3694,8 +3701,6 @@ static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz; hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf); } - - kfree(hr_cq); } static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index f07b2ec86ec2..3e45b119b0eb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -446,6 +446,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { .reg_user_mr = hns_roce_reg_user_mr, INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 205053cb5f97..3100b0c31b0a 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -1075,27 +1075,27 @@ static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) cq = &iwcq->sc_cq; i40iw_cq_wq_destroy(iwdev, cq); cq_free_resources(iwdev, iwcq); - kfree(iwcq); i40iw_rem_devusecount(iwdev); } /** * i40iw_create_cq - create cq - * @ibdev: device pointer from stack + * @ibcq: CQ allocated * @attr: attributes for cq * @udata: user data */ -static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int i40iw_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_cq *iwcq; + struct i40iw_cq *iwcq = to_iwcq(ibcq); struct i40iw_pbl *iwpbl; u32 cq_num = 0; struct i40iw_sc_cq *cq; struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cq_init_info info; + struct i40iw_cq_init_info info = {}; enum i40iw_status_code status; struct i40iw_cqp_request *cqp_request; struct cqp_commands_info *cqp_info; @@ -1105,22 +1105,16 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, int entries = attr->cqe; if (iwdev->closing) - return ERR_PTR(-ENODEV); + return -ENODEV; if (entries > iwdev->max_cqe) - return ERR_PTR(-EINVAL); - - iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL); - if (!iwcq) - return ERR_PTR(-ENOMEM); - - memset(&info, 0, sizeof(info)); + return -EINVAL; err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs, iwdev->max_cq, &cq_num, &iwdev->next_cq); if (err_code) - goto error; + return err_code; cq = &iwcq->sc_cq; cq->back_cq = (void *)iwcq; @@ -1227,15 +1221,13 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, } i40iw_add_devusecount(iwdev); - return (struct ib_cq *)iwcq; + return 0; cq_destroy: i40iw_cq_wq_destroy(iwdev, cq); cq_free_resources: cq_free_resources(iwdev, iwcq); -error: - kfree(iwcq); - return ERR_PTR(err_code); + return err_code; } /** @@ -2693,6 +2685,7 @@ static const struct ib_device_ops i40iw_dev_ops = { .reg_user_mr = i40iw_reg_user_mr, .req_notify_cq = i40iw_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 8eb7490dabb8..72f238ddafb5 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -172,14 +172,14 @@ err_buf: } #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_cq *cq; + struct mlx4_ib_cq *cq = to_mcq(ibcq); struct mlx4_uar *uar; void *buf_addr; int err; @@ -187,14 +187,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, udata, struct mlx4_ib_ucontext, ibucontext); if (entries < 1 || entries > dev->dev->caps.max_cqes) - return ERR_PTR(-EINVAL); + return -EINVAL; if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); + return -EINVAL; entries = roundup_pow_of_two(entries + 1); cq->ibcq.cqe = entries - 1; @@ -269,7 +265,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, goto err_cq_free; } - return &cq->ibcq; + return 0; err_cq_free: mlx4_cq_free(dev->dev, &cq->mcq); @@ -289,11 +285,8 @@ err_mtt: err_db: if (!udata) mlx4_db_free(dev->dev, &cq->db); - err_cq: - kfree(cq); - - return ERR_PTR(err); + return err; } static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, @@ -506,8 +499,6 @@ void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); } - - kfree(mcq); } static void dump_cqe(void *cqe) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 5d7a87842291..8790101facb7 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -2565,6 +2565,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = { .resize_cq = mlx4_ib_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mlx4_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext), diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index af5ee45a9f19..81b3d85e5167 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -743,9 +743,8 @@ int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index ebd01bd7f8f6..07b73df0e1a3 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -884,14 +884,14 @@ static void notify_soft_wc_handler(struct work_struct *work) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); } -struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx5_ib_dev *dev = to_mdev(ibdev); - struct mlx5_ib_cq *cq; + struct mlx5_ib_cq *cq = to_mcq(ibcq); int uninitialized_var(index); int uninitialized_var(inlen); u32 *cqb = NULL; @@ -903,18 +903,14 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, if (entries < 0 || (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))) - return ERR_PTR(-EINVAL); + return -EINVAL; if (check_cq_create_flags(attr->flags)) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; entries = roundup_pow_of_two(entries + 1); if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); + return -EINVAL; cq->ibcq.cqe = entries - 1; mutex_init(&cq->resize_mutex); @@ -929,13 +925,13 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size, &index, &inlen); if (err) - goto err_create; + return err; } else { cqe_size = cache_line_size() == 128 ? 128 : 64; err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index, &inlen); if (err) - goto err_create; + return err; INIT_WORK(&cq->notify_work, notify_soft_wc_handler); } @@ -980,7 +976,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, kvfree(cqb); - return &cq->ibcq; + return 0; err_cmd: mlx5_core_destroy_cq(dev->mdev, &cq->mcq); @@ -991,11 +987,7 @@ err_cqb: destroy_cq_user(cq, udata); else destroy_cq_kernel(dev, cq); - -err_create: - kfree(cq); - - return ERR_PTR(err); + return err; } void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) @@ -1008,8 +1000,6 @@ void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) destroy_cq_user(mcq, udata); else destroy_cq_kernel(dev, mcq); - - kfree(mcq); } static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 1e3d936ed809..99eb4a8b0b0d 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -4891,18 +4891,19 @@ static int create_dev_resources(struct mlx5_ib_resources *devr) if (ret) goto error0; - devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL); - if (IS_ERR(devr->c0)) { - ret = PTR_ERR(devr->c0); + devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq); + if (!devr->c0) { + ret = -ENOMEM; goto error1; } - devr->c0->device = &dev->ib_dev; - devr->c0->uobject = NULL; - devr->c0->comp_handler = NULL; - devr->c0->event_handler = NULL; - devr->c0->cq_context = NULL; + + devr->c0->device = &dev->ib_dev; atomic_set(&devr->c0->usecnt, 0); + ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL); + if (ret) + goto err_create_cq; + devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL); if (IS_ERR(devr->x0)) { ret = PTR_ERR(devr->x0); @@ -4994,6 +4995,8 @@ error3: mlx5_ib_dealloc_xrcd(devr->x0, NULL); error2: mlx5_ib_destroy_cq(devr->c0, NULL); +err_create_cq: + kfree(devr->c0); error1: mlx5_ib_dealloc_pd(devr->p0, NULL); error0: @@ -5012,6 +5015,7 @@ static void destroy_dev_resources(struct mlx5_ib_resources *devr) mlx5_ib_dealloc_xrcd(devr->x0, NULL); mlx5_ib_dealloc_xrcd(devr->x1, NULL); mlx5_ib_destroy_cq(devr->c0, NULL); + kfree(devr->c0); mlx5_ib_dealloc_pd(devr->p0, NULL); kfree(devr->p0); @@ -6182,6 +6186,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .resize_cq = mlx5_ib_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext), diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index bf697c5b3e79..f2ad0372d38d 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1115,9 +1115,8 @@ int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer, int buflen, size_t *bc); int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer, int buflen, size_t *bc); -struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 81fc04e1c142..efd4e3d13ae2 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -601,10 +601,11 @@ static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) return 0; } -static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int mthca_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; struct mthca_create_cq ucmd; struct mthca_cq *cq; @@ -614,20 +615,20 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, udata, struct mthca_ucontext, ibucontext); if (attr->flags) - return ERR_PTR(-EINVAL); + return -EINVAL; if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) - return ERR_PTR(-EFAULT); + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) + return -EFAULT; err = mthca_map_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.set_db_index, ucmd.set_db_page); if (err) - return ERR_PTR(err); + return err; err = mthca_map_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.arm_db_index, @@ -636,11 +637,7 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, goto err_unmap_set; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - err = -ENOMEM; - goto err_unmap_arm; - } + cq = to_mcq(ibcq); if (udata) { cq->buf.mr.ibmr.lkey = ucmd.lkey; @@ -655,20 +652,17 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, cq); if (err) - goto err_free; + goto err_unmap_arm; if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) { mthca_free_cq(to_mdev(ibdev), cq); err = -EFAULT; - goto err_free; + goto err_unmap_arm; } cq->resize_buf = NULL; - return &cq->ibcq; - -err_free: - kfree(cq); + return 0; err_unmap_arm: if (udata) @@ -680,7 +674,7 @@ err_unmap_set: mthca_unmap_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.set_db_index); - return ERR_PTR(err); + return err; } static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, @@ -823,7 +817,6 @@ static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) to_mcq(cq)->set_ci_db_index); } mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); - kfree(cq); } static inline u32 convert_access(int acc) @@ -1187,6 +1180,7 @@ static const struct ib_device_ops mthca_dev_ops = { .resize_cq = mthca_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index cac3fa624c4d..0420203820f6 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -1374,16 +1374,17 @@ static int nes_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) /** * nes_create_cq */ -static struct ib_cq *nes_create_cq(struct ib_device *ibdev, +static int nes_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; u64 u64temp; struct nes_vnic *nesvnic = to_nesvnic(ibdev); struct nes_device *nesdev = nesvnic->nesdev; struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_cq *nescq; + struct nes_cq *nescq = to_nescq(ibcq); struct nes_ucontext *nes_ucontext = NULL; struct nes_cqp_request *cqp_request; void *mem = NULL; @@ -1399,22 +1400,15 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, int ret; if (attr->flags) - return ERR_PTR(-EINVAL); + return -EINVAL; if (entries > nesadapter->max_cqe) - return ERR_PTR(-EINVAL); + return -EINVAL; err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ); - if (err) { - return ERR_PTR(err); - } - - nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL); - if (!nescq) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return ERR_PTR(-ENOMEM); - } + if (err) + return err; nescq->hw_cq.cq_size = max(entries + 1, 5); nescq->hw_cq.cq_number = cq_num; @@ -1424,10 +1418,10 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( udata, struct nes_ucontext, ibucontext); - if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) { + if (ib_copy_from_udata(&req, udata, + sizeof(struct nes_create_cq_req))) { nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); + return -EFAULT; } nesvnic->mcrq_ucontext = nes_ucontext; nes_ucontext->mcrqf = req.mcrqf; @@ -1441,8 +1435,6 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, nescq->mcrqf = nes_ucontext->mcrqf; nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); } - nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n", - (unsigned long)req.user_cq_buffer, entries); err = 1; list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) { if (nespbl->user_base == (unsigned long )req.user_cq_buffer) { @@ -1455,8 +1447,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, } if (err) { nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); + return -EFAULT; } pbl_entries = nespbl->pbl_size >> 3; @@ -1472,15 +1463,11 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, if (!mem) { printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n"); nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } nescq->hw_cq.cq_vbase = mem; nescq->hw_cq.cq_head = 0; - nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n", - nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase, - (u32)nescq->hw_cq.cq_pbase); } nescq->hw_cq.ce_handler = nes_iwarp_ce_handler; @@ -1500,8 +1487,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, } nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } cqp_request->waiting = 1; cqp_wqe = &cqp_request->cqp_wqe; @@ -1528,8 +1514,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, kfree(nespbl); } nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } else { opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK); nescq->virtual_cq = 2; @@ -1550,8 +1535,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, kfree(nespbl); } nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } else { opcode |= NES_CQP_CQ_VIRT; nescq->virtual_cq = 1; @@ -1607,8 +1591,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, kfree(nespbl); } nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EIO); + return -EIO; } nes_put_cqp_request(nesdev, cqp_request); @@ -1620,17 +1603,16 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev, resp.cq_id = nescq->hw_cq.cq_number; resp.cq_size = nescq->hw_cq.cq_size; resp.mmap_db_index = 0; - if (ib_copy_to_udata(udata, &resp, sizeof resp - sizeof resp.reserved)) { + if (ib_copy_to_udata(udata, &resp, + sizeof(resp) - sizeof(resp.reserved))) { nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); + return -EFAULT; } } - return &nescq->ibcq; + return 0; } - /** * nes_destroy_cq */ @@ -1700,7 +1682,6 @@ static void nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) if (nescq->cq_mem_size) pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); - kfree(nescq); } /** @@ -3584,6 +3565,7 @@ static const struct ib_device_ops nes_dev_ops = { .reg_user_mr = nes_reg_user_mr, .req_notify_cq = nes_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, nes_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index b326313d413f..c15cfc6cef81 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -182,6 +182,7 @@ static const struct ib_device_ops ocrdma_dev_ops = { .resize_cq = ocrdma_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, ocrdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, ocrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, ocrdma_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, ocrdma_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 94e4f7f9b1f7..10b35edb286b 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -977,12 +977,12 @@ err: return status; } -struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; - struct ocrdma_cq *cq; + struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); @@ -991,16 +991,13 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, struct ocrdma_create_cq_ureq ureq; if (attr->flags) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) - return ERR_PTR(-EFAULT); + return -EFAULT; } else ureq.dpp_cq = 0; - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); spin_lock_init(&cq->cq_lock); spin_lock_init(&cq->comp_handler_lock); @@ -1011,10 +1008,9 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, pd_id = uctx->cntxt_pd->id; status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); - if (status) { - kfree(cq); - return ERR_PTR(status); - } + if (status) + return status; + if (udata) { status = ocrdma_copy_cq_uresp(dev, cq, udata); if (status) @@ -1022,12 +1018,11 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, } cq->phase = OCRDMA_CQE_VALID; dev->cq_tbl[cq->id] = cq; - return &cq->ibcq; + return 0; ctx_err: ocrdma_mbx_destroy_cq(dev, cq); - kfree(cq); - return ERR_PTR(status); + return status; } int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, @@ -1095,8 +1090,6 @@ void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) ocrdma_get_db_addr(dev, pdid), dev->nic_info.db_page_size); } - - kfree(cq); } static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index 89cebe05669e..32488da1b752 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -71,9 +71,8 @@ int ocrdma_mmap(struct ib_ucontext *, struct vm_area_struct *vma); int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); void ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); -struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index a0bb07ba0f3c..a0a7ba0a5af4 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -224,6 +224,7 @@ static const struct ib_device_ops qedr_dev_ops = { .resize_cq = qedr_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, qedr_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext), diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index be29bbbc4b14..3fc7a4e901c3 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -806,20 +806,20 @@ int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) return 0; } -struct ib_cq *qedr_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; struct qedr_ucontext *ctx = rdma_udata_to_drv_context( udata, struct qedr_ucontext, ibucontext); struct qed_rdma_destroy_cq_out_params destroy_oparams; struct qed_rdma_destroy_cq_in_params destroy_iparams; struct qedr_dev *dev = get_qedr_dev(ibdev); struct qed_rdma_create_cq_in_params params; - struct qedr_create_cq_ureq ureq; + struct qedr_create_cq_ureq ureq = {}; int vector = attr->comp_vector; int entries = attr->cqe; - struct qedr_cq *cq; + struct qedr_cq *cq = get_qedr_cq(ibcq); int chain_entries; int page_cnt; u64 pbl_ptr; @@ -834,18 +834,13 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev, DP_ERR(dev, "create cq: the number of entries %d is too high. Must be equal or below %d.\n", entries, QEDR_MAX_CQES); - return ERR_PTR(-EINVAL); + return -EINVAL; } chain_entries = qedr_align_cq_entries(entries); chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); - if (udata) { - memset(&ureq, 0, sizeof(ureq)); if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { DP_ERR(dev, "create cq: problem copying data from user space\n"); @@ -923,7 +918,7 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev, "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", cq->icid, cq, params.cq_size); - return &cq->ibcq; + return 0; err3: destroy_iparams.icid = cq->icid; @@ -938,8 +933,7 @@ err1: if (udata) ib_umem_release(cq->q.umem); err0: - kfree(cq); - return ERR_PTR(-EINVAL); + return -EINVAL; } int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) @@ -969,7 +963,7 @@ void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) /* GSIs CQs are handled by driver, so they don't exist in the FW */ if (cq->cq_type == QEDR_CQ_TYPE_GSI) - goto done; + return; iparams.icid = cq->icid; dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); @@ -1008,10 +1002,6 @@ void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) * Since the destroy CQ ramrod has also been received on the EQ we can * be certain that there's no event handler in process. */ -done: - cq->sig = ~cq->sig; - - kfree(cq); } static inline int get_gid_info_from_table(struct ib_qp *ibqp, diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 32d7ce77e339..9aaa90283d6e 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -50,9 +50,8 @@ int qedr_mmap(struct ib_ucontext *, struct vm_area_struct *vma); int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); -struct ib_cq *qedr_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); diff --git a/drivers/infiniband/hw/usnic/usnic_ib.h b/drivers/infiniband/hw/usnic/usnic_ib.h index 525bf272671e..84dd682d2334 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib.h +++ b/drivers/infiniband/hw/usnic/usnic_ib.h @@ -61,6 +61,10 @@ struct usnic_ib_pd { struct usnic_uiom_pd *umem_pd; }; +struct usnic_ib_cq { + struct ib_cq ibcq; +}; + struct usnic_ib_mr { struct ib_mr ibmr; struct usnic_uiom_reg *umem; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index e701322dc740..6ae5ce007fed 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -354,6 +354,7 @@ static const struct ib_device_ops usnic_dev_ops = { .query_qp = usnic_ib_query_qp, .reg_user_mr = usnic_ib_reg_mr, INIT_RDMA_OBJ_SIZE(ib_pd, usnic_ib_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, usnic_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, usnic_ib_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 5686d14b86fe..eeb07b245ef9 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -587,26 +587,18 @@ out_unlock: return status; } -struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct ib_cq *cq; - - usnic_dbg("\n"); if (attr->flags) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-EBUSY); + return -EINVAL; - return cq; + return 0; } void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { - kfree(cq); + return; } struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 0b9d993433a7..2aedf78c13cf 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -58,9 +58,8 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); -struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index 0682781f6555..38573fc0a9bf 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -92,20 +92,19 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq, /** * pvrdma_create_cq - create completion queue - * @ibdev: the device + * @ibcq: Allocated CQ * @attr: completion queue attributes * @udata: user data * - * @return: ib_cq completion queue pointer on success, - * otherwise returns negative errno. + * @return: 0 on success */ -struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; struct pvrdma_dev *dev = to_vdev(ibdev); - struct pvrdma_cq *cq; + struct pvrdma_cq *cq = to_vcq(ibcq); int ret; int npages; unsigned long flags; @@ -113,7 +112,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, union pvrdma_cmd_resp rsp; struct pvrdma_cmd_create_cq *cmd = &req.create_cq; struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp; - struct pvrdma_create_cq_resp cq_resp = {0}; + struct pvrdma_create_cq_resp cq_resp = {}; struct pvrdma_create_cq ucmd; struct pvrdma_ucontext *context = rdma_udata_to_drv_context( udata, struct pvrdma_ucontext, ibucontext); @@ -122,16 +121,10 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, entries = roundup_pow_of_two(entries); if (entries < 1 || entries > dev->dsr->caps.max_cqe) - return ERR_PTR(-EINVAL); + return -EINVAL; if (!atomic_add_unless(&dev->num_cqs, 1, dev->dsr->caps.max_cq)) - return ERR_PTR(-ENOMEM); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - atomic_dec(&dev->num_cqs); - return ERR_PTR(-ENOMEM); - } + return -ENOMEM; cq->ibcq.cqe = entries; cq->is_kernel = !udata; @@ -211,11 +204,11 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, dev_warn(&dev->pdev->dev, "failed to copy back udata\n"); pvrdma_destroy_cq(&cq->ibcq, udata); - return ERR_PTR(-EINVAL); + return -EINVAL; } } - return &cq->ibcq; + return 0; err_page_dir: pvrdma_page_dir_cleanup(dev, &cq->pdir); @@ -224,9 +217,7 @@ err_umem: ib_umem_release(cq->umem); err_cq: atomic_dec(&dev->num_cqs); - kfree(cq); - - return ERR_PTR(ret); + return ret; } static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) @@ -239,7 +230,6 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) ib_umem_release(cq->umem); pvrdma_page_dir_cleanup(dev, &cq->pdir); - kfree(cq); } /** diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 0c48464ffff1..e580ae9cc55a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -182,6 +182,7 @@ static const struct ib_device_ops pvrdma_dev_ops = { .req_notify_cq = pvrdma_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_ah, pvrdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, pvrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, pvrdma_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, pvrdma_ucontext, ibucontext), }; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index f0dd6e4d058b..e4a48f5c0c85 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -409,9 +409,8 @@ struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata); int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); -struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); -- cgit v1.2.3 From 2d3c72ed504196edb2f22a08cb03a0f9fb7e564f Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Mon, 10 Jun 2019 16:49:11 -0300 Subject: rdma: Remove nes This driver was first merged over 10 years ago and has not seen major activity by the authors in the last 7 years. However, in that time it has been patched 150 times to adapt it to changing kernel APIs. Further, the hardware has several issues, like not supporting 64 bit DMA, that make it rather uninteresting for use with modern systems and RDMA. Signed-off-by: Jason Gunthorpe Reviewed-by: Leon Romanovsky Reviewed-by: Shiraz Saleem Signed-off-by: Doug Ledford --- drivers/infiniband/hw/Makefile | 1 - drivers/infiniband/hw/nes/Kconfig | 15 - drivers/infiniband/hw/nes/Makefile | 3 - drivers/infiniband/hw/nes/nes.c | 1205 ---------- drivers/infiniband/hw/nes/nes.h | 574 ----- drivers/infiniband/hw/nes/nes_cm.c | 3992 ------------------------------- drivers/infiniband/hw/nes/nes_cm.h | 470 ---- drivers/infiniband/hw/nes/nes_context.h | 193 -- drivers/infiniband/hw/nes/nes_hw.c | 3887 ------------------------------ drivers/infiniband/hw/nes/nes_hw.h | 1380 ----------- drivers/infiniband/hw/nes/nes_mgt.c | 1155 --------- drivers/infiniband/hw/nes/nes_mgt.h | 97 - drivers/infiniband/hw/nes/nes_nic.c | 1870 --------------- drivers/infiniband/hw/nes/nes_utils.c | 913 ------- drivers/infiniband/hw/nes/nes_verbs.c | 3721 ---------------------------- drivers/infiniband/hw/nes/nes_verbs.h | 198 -- 16 files changed, 19674 deletions(-) delete mode 100644 drivers/infiniband/hw/nes/Kconfig delete mode 100644 drivers/infiniband/hw/nes/Makefile delete mode 100644 drivers/infiniband/hw/nes/nes.c delete mode 100644 drivers/infiniband/hw/nes/nes.h delete mode 100644 drivers/infiniband/hw/nes/nes_cm.c delete mode 100644 drivers/infiniband/hw/nes/nes_cm.h delete mode 100644 drivers/infiniband/hw/nes/nes_context.h delete mode 100644 drivers/infiniband/hw/nes/nes_hw.c delete mode 100644 drivers/infiniband/hw/nes/nes_hw.h delete mode 100644 drivers/infiniband/hw/nes/nes_mgt.c delete mode 100644 drivers/infiniband/hw/nes/nes_mgt.h delete mode 100644 drivers/infiniband/hw/nes/nes_nic.c delete mode 100644 drivers/infiniband/hw/nes/nes_utils.c delete mode 100644 drivers/infiniband/hw/nes/nes_verbs.c delete mode 100644 drivers/infiniband/hw/nes/nes_verbs.h (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 77094be1b262..433fca59febd 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_INFINIBAND_EFA) += efa/ obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/ obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/ obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/ -obj-$(CONFIG_INFINIBAND_NES) += nes/ obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/ obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA) += vmw_pvrdma/ obj-$(CONFIG_INFINIBAND_USNIC) += usnic/ diff --git a/drivers/infiniband/hw/nes/Kconfig b/drivers/infiniband/hw/nes/Kconfig deleted file mode 100644 index 52caae954e4a..000000000000 --- a/drivers/infiniband/hw/nes/Kconfig +++ /dev/null @@ -1,15 +0,0 @@ -config INFINIBAND_NES - tristate "NetEffect RNIC Driver" - depends on PCI && INET - select LIBCRC32C - ---help--- - This is the RDMA Network Interface Card (RNIC) driver for - NetEffect Ethernet Cluster Server Adapters. - -config INFINIBAND_NES_DEBUG - bool "Verbose debugging output" - depends on INFINIBAND_NES - default n - ---help--- - This option enables debug messages from the NetEffect RNIC - driver. Select this if you are diagnosing a problem. diff --git a/drivers/infiniband/hw/nes/Makefile b/drivers/infiniband/hw/nes/Makefile deleted file mode 100644 index 97820c23ecef..000000000000 --- a/drivers/infiniband/hw/nes/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -obj-$(CONFIG_INFINIBAND_NES) += iw_nes.o - -iw_nes-objs := nes.o nes_hw.o nes_nic.o nes_utils.o nes_verbs.o nes_cm.o nes_mgt.o diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c deleted file mode 100644 index e00add6d78ec..000000000000 --- a/drivers/infiniband/hw/nes/nes.c +++ /dev/null @@ -1,1205 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nes.h" - -#include -#include -#include -#include - -MODULE_AUTHOR("NetEffect"); -MODULE_DESCRIPTION("NetEffect RNIC Low-level iWARP Driver"); -MODULE_LICENSE("Dual BSD/GPL"); - -int interrupt_mod_interval = 0; - -/* Interoperability */ -int mpa_version = 1; -module_param(mpa_version, int, 0644); -MODULE_PARM_DESC(mpa_version, "MPA version to be used int MPA Req/Resp (0 or 1)"); - -/* Interoperability */ -int disable_mpa_crc = 0; -module_param(disable_mpa_crc, int, 0644); -MODULE_PARM_DESC(disable_mpa_crc, "Disable checking of MPA CRC"); - -unsigned int nes_drv_opt = NES_DRV_OPT_DISABLE_INT_MOD | NES_DRV_OPT_ENABLE_PAU; -module_param(nes_drv_opt, int, 0644); -MODULE_PARM_DESC(nes_drv_opt, "Driver option parameters"); - -unsigned int nes_debug_level = 0; -module_param_named(debug_level, nes_debug_level, uint, 0644); -MODULE_PARM_DESC(debug_level, "Enable debug output level"); - -unsigned int wqm_quanta = 0x10000; -module_param(wqm_quanta, int, 0644); -MODULE_PARM_DESC(wqm_quanta, "WQM quanta"); - -static bool limit_maxrdreqsz; -module_param(limit_maxrdreqsz, bool, 0644); -MODULE_PARM_DESC(limit_maxrdreqsz, "Limit max read request size to 256 Bytes"); - -LIST_HEAD(nes_adapter_list); -static LIST_HEAD(nes_dev_list); - -atomic_t qps_destroyed; - -static unsigned int ee_flsh_adapter; -static unsigned int sysfs_nonidx_addr; -static unsigned int sysfs_idx_addr; - -static const struct pci_device_id nes_pci_table[] = { - { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020), }, - { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR), }, - {0} -}; - -MODULE_DEVICE_TABLE(pci, nes_pci_table); - -static int nes_inetaddr_event(struct notifier_block *, unsigned long, void *); -static int nes_net_event(struct notifier_block *, unsigned long, void *); -static int nes_notifiers_registered; - - -static struct notifier_block nes_inetaddr_notifier = { - .notifier_call = nes_inetaddr_event -}; - -static struct notifier_block nes_net_notifier = { - .notifier_call = nes_net_event -}; - -/** - * nes_inetaddr_event - */ -static int nes_inetaddr_event(struct notifier_block *notifier, - unsigned long event, void *ptr) -{ - struct in_ifaddr *ifa = ptr; - struct net_device *event_netdev = ifa->ifa_dev->dev; - struct nes_device *nesdev; - struct net_device *netdev; - struct net_device *upper_dev; - struct nes_vnic *nesvnic; - unsigned int is_bonded; - - nes_debug(NES_DBG_NETDEV, "nes_inetaddr_event: ip address %pI4, netmask %pI4.\n", - &ifa->ifa_address, &ifa->ifa_mask); - list_for_each_entry(nesdev, &nes_dev_list, list) { - nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p. (%s)\n", - nesdev, nesdev->netdev[0]->name); - netdev = nesdev->netdev[0]; - nesvnic = netdev_priv(netdev); - upper_dev = netdev_master_upper_dev_get(netdev); - is_bonded = netif_is_bond_slave(netdev) && - (upper_dev == event_netdev); - if ((netdev == event_netdev) || is_bonded) { - if (nesvnic->rdma_enabled == 0) { - nes_debug(NES_DBG_NETDEV, "Returning without processing event for %s since" - " RDMA is not enabled.\n", - netdev->name); - return NOTIFY_OK; - } - /* we have ifa->ifa_address/mask here if we need it */ - switch (event) { - case NETDEV_DOWN: - nes_debug(NES_DBG_NETDEV, "event:DOWN\n"); - nes_write_indexed(nesdev, - NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)), 0); - - nes_manage_arp_cache(netdev, netdev->dev_addr, - ntohl(nesvnic->local_ipaddr), NES_ARP_DELETE); - nesvnic->local_ipaddr = 0; - if (is_bonded) - continue; - else - return NOTIFY_OK; - break; - case NETDEV_UP: - nes_debug(NES_DBG_NETDEV, "event:UP\n"); - - if (nesvnic->local_ipaddr != 0) { - nes_debug(NES_DBG_NETDEV, "Interface already has local_ipaddr\n"); - return NOTIFY_OK; - } - /* fall through */ - case NETDEV_CHANGEADDR: - /* Add the address to the IP table */ - if (upper_dev) { - struct in_device *in; - - rcu_read_lock(); - in = __in_dev_get_rcu(upper_dev); - nesvnic->local_ipaddr = in->ifa_list->ifa_address; - rcu_read_unlock(); - } else { - nesvnic->local_ipaddr = ifa->ifa_address; - } - - nes_write_indexed(nesdev, - NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)), - ntohl(nesvnic->local_ipaddr)); - nes_manage_arp_cache(netdev, netdev->dev_addr, - ntohl(nesvnic->local_ipaddr), NES_ARP_ADD); - if (is_bonded) - continue; - else - return NOTIFY_OK; - break; - default: - break; - } - } - } - - return NOTIFY_DONE; -} - - -/** - * nes_net_event - */ -static int nes_net_event(struct notifier_block *notifier, - unsigned long event, void *ptr) -{ - struct neighbour *neigh = ptr; - struct nes_device *nesdev; - struct net_device *netdev; - struct nes_vnic *nesvnic; - - switch (event) { - case NETEVENT_NEIGH_UPDATE: - list_for_each_entry(nesdev, &nes_dev_list, list) { - /* nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p.\n", nesdev); */ - netdev = nesdev->netdev[0]; - nesvnic = netdev_priv(netdev); - if (netdev == neigh->dev) { - if (nesvnic->rdma_enabled == 0) { - nes_debug(NES_DBG_NETDEV, "Skipping device %s since no RDMA\n", - netdev->name); - } else { - if (neigh->nud_state & NUD_VALID) { - nes_manage_arp_cache(neigh->dev, neigh->ha, - ntohl(*(__be32 *)neigh->primary_key), NES_ARP_ADD); - } else { - nes_manage_arp_cache(neigh->dev, neigh->ha, - ntohl(*(__be32 *)neigh->primary_key), NES_ARP_DELETE); - } - } - return NOTIFY_OK; - } - } - break; - default: - nes_debug(NES_DBG_NETDEV, "NETEVENT_ %lu undefined\n", event); - break; - } - - return NOTIFY_DONE; -} - - -/** - * nes_add_ref - */ -void nes_add_ref(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp; - - nesqp = to_nesqp(ibqp); - nes_debug(NES_DBG_QP, "Bumping refcount for QP%u. Pre-inc value = %u\n", - ibqp->qp_num, atomic_read(&nesqp->refcount)); - atomic_inc(&nesqp->refcount); -} - -static void nes_cqp_rem_ref_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - unsigned long flags; - struct nes_qp *nesqp = cqp_request->cqp_callback_pointer; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - atomic_inc(&qps_destroyed); - - /* Free the control structures */ - - if (nesqp->pbl_vbase) { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); - nesqp->pbl_vbase = NULL; - - } else { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); - } - nes_free_resource(nesadapter, nesadapter->allocated_qps, nesqp->hwqp.qp_id); - - nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = NULL; - kfree(nesqp->allocated_buffer); - -} - -/** - * nes_rem_ref - */ -void nes_rem_ref(struct ib_qp *ibqp) -{ - u64 u64temp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - u32 opcode; - - nesqp = to_nesqp(ibqp); - - if (atomic_read(&nesqp->refcount) == 0) { - printk(KERN_INFO PFX "%s: Reference count already 0 for QP%d, last aeq = 0x%04X.\n", - __func__, ibqp->qp_num, nesqp->last_aeq); - BUG(); - } - - if (atomic_dec_and_test(&nesqp->refcount)) { - if (nesqp->pau_mode) - nes_destroy_pau_qp(nesdev, nesqp); - - /* Destroy the QP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return; - } - cqp_request->waiting = 0; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_cqp_rem_ref_callback; - cqp_request->cqp_callback_pointer = nesqp; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - opcode = NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_IWARP; - - if (nesqp->hte_added) { - opcode |= NES_CQP_QP_DEL_HTE; - nesqp->hte_added = 0; - } - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - nes_post_cqp_request(nesdev, cqp_request); - } -} - - -/** - * nes_get_qp - */ -struct ib_qp *nes_get_qp(struct ib_device *device, int qpn) -{ - struct nes_vnic *nesvnic = to_nesvnic(device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if ((qpn < NES_FIRST_QPN) || (qpn >= (NES_FIRST_QPN + nesadapter->max_qp))) - return NULL; - - return &nesadapter->qp_table[qpn - NES_FIRST_QPN]->ibqp; -} - - -/** - * nes_print_macaddr - */ -static void nes_print_macaddr(struct net_device *netdev) -{ - nes_debug(NES_DBG_INIT, "%s: %pM, IRQ %u\n", - netdev->name, netdev->dev_addr, netdev->irq); -} - -/** - * nes_interrupt - handle interrupts - */ -static irqreturn_t nes_interrupt(int irq, void *dev_id) -{ - struct nes_device *nesdev = (struct nes_device *)dev_id; - int handled = 0; - u32 int_mask; - u32 int_req; - u32 int_stat; - u32 intf_int_stat; - u32 timer_stat; - - if (nesdev->msi_enabled) { - /* No need to read the interrupt pending register if msi is enabled */ - handled = 1; - } else { - if (unlikely(nesdev->nesadapter->hw_rev == NE020_REV)) { - /* Master interrupt enable provides synchronization for kicking off bottom half - when interrupt sharing is going on */ - int_mask = nes_read32(nesdev->regs + NES_INT_MASK); - if (int_mask & 0x80000000) { - /* Check interrupt status to see if this might be ours */ - int_stat = nes_read32(nesdev->regs + NES_INT_STAT); - int_req = nesdev->int_req; - if (int_stat&int_req) { - /* if interesting CEQ or AEQ is pending, claim the interrupt */ - if ((int_stat&int_req) & (~(NES_INT_TIMER|NES_INT_INTF))) { - handled = 1; - } else { - if (((int_stat & int_req) & NES_INT_TIMER) == NES_INT_TIMER) { - /* Timer might be running but might be for another function */ - timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT); - if ((timer_stat & nesdev->timer_int_req) != 0) { - handled = 1; - } - } - if ((((int_stat & int_req) & NES_INT_INTF) == NES_INT_INTF) && - (handled == 0)) { - intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT); - if ((intf_int_stat & nesdev->intf_int_req) != 0) { - handled = 1; - } - } - } - if (handled) { - nes_write32(nesdev->regs+NES_INT_MASK, int_mask & (~0x80000000)); - int_mask = nes_read32(nesdev->regs+NES_INT_MASK); - /* Save off the status to save an additional read */ - nesdev->int_stat = int_stat; - nesdev->napi_isr_ran = 1; - } - } - } - } else { - handled = nes_read32(nesdev->regs+NES_INT_PENDING); - } - } - - if (handled) { - - if (nes_napi_isr(nesdev) == 0) { - tasklet_schedule(&nesdev->dpc_tasklet); - - } - return IRQ_HANDLED; - } else { - return IRQ_NONE; - } -} - - -/** - * nes_probe - Device initialization - */ -static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent) -{ - struct net_device *netdev = NULL; - struct nes_device *nesdev = NULL; - int ret = 0; - void __iomem *mmio_regs = NULL; - u8 hw_rev; - - printk(KERN_INFO PFX "NetEffect RNIC driver v%s loading. (%s)\n", - DRV_VERSION, pci_name(pcidev)); - - ret = pci_enable_device(pcidev); - if (ret) { - printk(KERN_ERR PFX "Unable to enable PCI device. (%s)\n", pci_name(pcidev)); - goto bail0; - } - - nes_debug(NES_DBG_INIT, "BAR0 (@0x%08lX) size = 0x%lX bytes\n", - (long unsigned int)pci_resource_start(pcidev, BAR_0), - (long unsigned int)pci_resource_len(pcidev, BAR_0)); - nes_debug(NES_DBG_INIT, "BAR1 (@0x%08lX) size = 0x%lX bytes\n", - (long unsigned int)pci_resource_start(pcidev, BAR_1), - (long unsigned int)pci_resource_len(pcidev, BAR_1)); - - /* Make sure PCI base addr are MMIO */ - if (!(pci_resource_flags(pcidev, BAR_0) & IORESOURCE_MEM) || - !(pci_resource_flags(pcidev, BAR_1) & IORESOURCE_MEM)) { - printk(KERN_ERR PFX "PCI regions not an MMIO resource\n"); - ret = -ENODEV; - goto bail1; - } - - /* Reserve PCI I/O and memory resources */ - ret = pci_request_regions(pcidev, DRV_NAME); - if (ret) { - printk(KERN_ERR PFX "Unable to request regions. (%s)\n", pci_name(pcidev)); - goto bail1; - } - - if ((sizeof(dma_addr_t) > 4)) { - ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64)); - if (ret < 0) { - printk(KERN_ERR PFX "64b DMA mask configuration failed\n"); - goto bail2; - } - ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64)); - if (ret) { - printk(KERN_ERR PFX "64b DMA consistent mask configuration failed\n"); - goto bail2; - } - } else { - ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32)); - if (ret < 0) { - printk(KERN_ERR PFX "32b DMA mask configuration failed\n"); - goto bail2; - } - ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); - if (ret) { - printk(KERN_ERR PFX "32b DMA consistent mask configuration failed\n"); - goto bail2; - } - } - - pci_set_master(pcidev); - - /* Allocate hardware structure */ - nesdev = kzalloc(sizeof(struct nes_device), GFP_KERNEL); - if (!nesdev) { - ret = -ENOMEM; - goto bail2; - } - - nes_debug(NES_DBG_INIT, "Allocated nes device at %p\n", nesdev); - nesdev->pcidev = pcidev; - pci_set_drvdata(pcidev, nesdev); - - pci_read_config_byte(pcidev, 0x0008, &hw_rev); - nes_debug(NES_DBG_INIT, "hw_rev=%u\n", hw_rev); - - spin_lock_init(&nesdev->indexed_regs_lock); - - /* Remap the PCI registers in adapter BAR0 to kernel VA space */ - mmio_regs = ioremap_nocache(pci_resource_start(pcidev, BAR_0), - pci_resource_len(pcidev, BAR_0)); - if (mmio_regs == NULL) { - printk(KERN_ERR PFX "Unable to remap BAR0\n"); - ret = -EIO; - goto bail3; - } - nesdev->regs = mmio_regs; - nesdev->index_reg = 0x50 + (PCI_FUNC(pcidev->devfn)*8) + mmio_regs; - - /* Ensure interrupts are disabled */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff); - - if (nes_drv_opt & NES_DRV_OPT_ENABLE_MSI) { - if (!pci_enable_msi(nesdev->pcidev)) { - nesdev->msi_enabled = 1; - nes_debug(NES_DBG_INIT, "MSI is enabled for device %s\n", - pci_name(pcidev)); - } else { - nes_debug(NES_DBG_INIT, "MSI is disabled by linux for device %s\n", - pci_name(pcidev)); - } - } else { - nes_debug(NES_DBG_INIT, "MSI not requested due to driver options for device %s\n", - pci_name(pcidev)); - } - - nesdev->csr_start = pci_resource_start(nesdev->pcidev, BAR_0); - nesdev->doorbell_region = pci_resource_start(nesdev->pcidev, BAR_1); - - /* Init the adapter */ - nesdev->nesadapter = nes_init_adapter(nesdev, hw_rev); - if (!nesdev->nesadapter) { - printk(KERN_ERR PFX "Unable to initialize adapter.\n"); - ret = -ENOMEM; - goto bail5; - } - nesdev->nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval; - nesdev->nesadapter->wqm_quanta = wqm_quanta; - - /* nesdev->base_doorbell_index = - nesdev->nesadapter->pd_config_base[PCI_FUNC(nesdev->pcidev->devfn)]; */ - nesdev->base_doorbell_index = 1; - nesdev->doorbell_start = nesdev->nesadapter->doorbell_start; - if (nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) { - switch (PCI_FUNC(nesdev->pcidev->devfn) % - nesdev->nesadapter->port_count) { - case 1: - nesdev->mac_index = 2; - break; - case 2: - nesdev->mac_index = 1; - break; - case 3: - nesdev->mac_index = 3; - break; - case 0: - default: - nesdev->mac_index = 0; - } - } else { - nesdev->mac_index = PCI_FUNC(nesdev->pcidev->devfn) % - nesdev->nesadapter->port_count; - } - - if ((limit_maxrdreqsz || - ((nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_GLADIUS) && - (hw_rev == NE020_REV1))) && - (pcie_get_readrq(pcidev) > 256)) { - if (pcie_set_readrq(pcidev, 256)) - printk(KERN_ERR PFX "Unable to set max read request" - " to 256 bytes\n"); - else - nes_debug(NES_DBG_INIT, "Max read request size set" - " to 256 bytes\n"); - } - - tasklet_init(&nesdev->dpc_tasklet, nes_dpc, (unsigned long)nesdev); - - /* bring up the Control QP */ - if (nes_init_cqp(nesdev)) { - ret = -ENODEV; - goto bail6; - } - - /* Arm the CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - PCI_FUNC(nesdev->pcidev->devfn)); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - - /* Enable the interrupts */ - nesdev->int_req = (0x101 << PCI_FUNC(nesdev->pcidev->devfn)) | - (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16)); - if (PCI_FUNC(nesdev->pcidev->devfn) < 4) { - nesdev->int_req |= (1 << (PCI_FUNC(nesdev->mac_index)+24)); - } - - /* TODO: This really should be the first driver to load, not function 0 */ - if (PCI_FUNC(nesdev->pcidev->devfn) == 0) { - /* pick up PCI and critical errors if the first driver to load */ - nesdev->intf_int_req = NES_INTF_INT_PCIERR | NES_INTF_INT_CRITERR; - nesdev->int_req |= NES_INT_INTF; - } else { - nesdev->intf_int_req = 0; - } - nesdev->intf_int_req |= (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16)); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0, 0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS2, 0x00001265); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS4, 0x18021804); - - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS3, 0x17801790); - - /* deal with both periodic and one_shot */ - nesdev->timer_int_req = 0x101 << PCI_FUNC(nesdev->pcidev->devfn); - nesdev->nesadapter->timer_int_req |= nesdev->timer_int_req; - nes_debug(NES_DBG_INIT, "setting int_req for function %u, nesdev = 0x%04X, adapter = 0x%04X\n", - PCI_FUNC(nesdev->pcidev->devfn), - nesdev->timer_int_req, nesdev->nesadapter->timer_int_req); - - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - - list_add_tail(&nesdev->list, &nes_dev_list); - - /* Request an interrupt line for the driver */ - ret = request_irq(pcidev->irq, nes_interrupt, IRQF_SHARED, DRV_NAME, nesdev); - if (ret) { - printk(KERN_ERR PFX "%s: requested IRQ %u is busy\n", - pci_name(pcidev), pcidev->irq); - goto bail65; - } - - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - - if (nes_notifiers_registered == 0) { - register_inetaddr_notifier(&nes_inetaddr_notifier); - register_netevent_notifier(&nes_net_notifier); - } - nes_notifiers_registered++; - - INIT_DELAYED_WORK(&nesdev->work, nes_recheck_link_status); - - /* Initialize network devices */ - netdev = nes_netdev_init(nesdev, mmio_regs); - if (netdev == NULL) { - ret = -ENOMEM; - goto bail7; - } - - /* Register network device */ - ret = register_netdev(netdev); - if (ret) { - printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n", ret); - nes_netdev_destroy(netdev); - goto bail7; - } - - nes_print_macaddr(netdev); - - nesdev->netdev_count++; - nesdev->nesadapter->netdev_count++; - - printk(KERN_INFO PFX "%s: NetEffect RNIC driver successfully loaded.\n", - pci_name(pcidev)); - return 0; - - bail7: - printk(KERN_ERR PFX "bail7\n"); - while (nesdev->netdev_count > 0) { - nesdev->netdev_count--; - nesdev->nesadapter->netdev_count--; - - unregister_netdev(nesdev->netdev[nesdev->netdev_count]); - nes_netdev_destroy(nesdev->netdev[nesdev->netdev_count]); - } - - nes_debug(NES_DBG_INIT, "netdev_count=%d, nesadapter->netdev_count=%d\n", - nesdev->netdev_count, nesdev->nesadapter->netdev_count); - - nes_notifiers_registered--; - if (nes_notifiers_registered == 0) { - unregister_netevent_notifier(&nes_net_notifier); - unregister_inetaddr_notifier(&nes_inetaddr_notifier); - } - - list_del(&nesdev->list); - nes_destroy_cqp(nesdev); - - bail65: - printk(KERN_ERR PFX "bail65\n"); - free_irq(pcidev->irq, nesdev); - if (nesdev->msi_enabled) { - pci_disable_msi(pcidev); - } - bail6: - printk(KERN_ERR PFX "bail6\n"); - tasklet_kill(&nesdev->dpc_tasklet); - /* Deallocate the Adapter Structure */ - nes_destroy_adapter(nesdev->nesadapter); - - bail5: - printk(KERN_ERR PFX "bail5\n"); - iounmap(nesdev->regs); - - bail3: - printk(KERN_ERR PFX "bail3\n"); - kfree(nesdev); - - bail2: - pci_release_regions(pcidev); - - bail1: - pci_disable_device(pcidev); - - bail0: - return ret; -} - - -/** - * nes_remove - unload from kernel - */ -static void nes_remove(struct pci_dev *pcidev) -{ - struct nes_device *nesdev = pci_get_drvdata(pcidev); - struct net_device *netdev; - int netdev_index = 0; - unsigned long flags; - - if (nesdev->netdev_count) { - netdev = nesdev->netdev[netdev_index]; - if (netdev) { - netif_stop_queue(netdev); - unregister_netdev(netdev); - nes_netdev_destroy(netdev); - - nesdev->netdev[netdev_index] = NULL; - nesdev->netdev_count--; - nesdev->nesadapter->netdev_count--; - } - } - - nes_notifiers_registered--; - if (nes_notifiers_registered == 0) { - unregister_netevent_notifier(&nes_net_notifier); - unregister_inetaddr_notifier(&nes_inetaddr_notifier); - } - - list_del(&nesdev->list); - nes_destroy_cqp(nesdev); - - free_irq(pcidev->irq, nesdev); - tasklet_kill(&nesdev->dpc_tasklet); - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - if (nesdev->link_recheck) { - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - cancel_delayed_work_sync(&nesdev->work); - } else { - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - } - - /* Deallocate the Adapter Structure */ - nes_destroy_adapter(nesdev->nesadapter); - - if (nesdev->msi_enabled) { - pci_disable_msi(pcidev); - } - - iounmap(nesdev->regs); - kfree(nesdev); - - /* nes_debug(NES_DBG_SHUTDOWN, "calling pci_release_regions.\n"); */ - pci_release_regions(pcidev); - pci_disable_device(pcidev); - pci_set_drvdata(pcidev, NULL); -} - - -static ssize_t adapter_show(struct device_driver *ddp, char *buf) -{ - unsigned int devfn = 0xffffffff; - unsigned char bus_number = 0xff; - unsigned int i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - devfn = nesdev->pcidev->devfn; - bus_number = nesdev->pcidev->bus->number; - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "%x:%x\n", bus_number, devfn); -} - -static ssize_t adapter_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - ee_flsh_adapter = simple_strtoul(p, &p, 10); - return strnlen(buf, count); -} - -static ssize_t eeprom_cmd_show(struct device_driver *ddp, char *buf) -{ - u32 eeprom_cmd = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - eeprom_cmd = nes_read32(nesdev->regs + NES_EEPROM_COMMAND); - break; - } - i++; - } - return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_cmd); -} - -static ssize_t eeprom_cmd_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_EEPROM_COMMAND, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t eeprom_data_show(struct device_driver *ddp, char *buf) -{ - u32 eeprom_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - eeprom_data = nes_read32(nesdev->regs + NES_EEPROM_DATA); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_data); -} - -static ssize_t eeprom_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_EEPROM_DATA, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t flash_cmd_show(struct device_driver *ddp, char *buf) -{ - u32 flash_cmd = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - flash_cmd = nes_read32(nesdev->regs + NES_FLASH_COMMAND); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_cmd); -} - -static ssize_t flash_cmd_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_FLASH_COMMAND, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t flash_data_show(struct device_driver *ddp, char *buf) -{ - u32 flash_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - flash_data = nes_read32(nesdev->regs + NES_FLASH_DATA); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_data); -} - -static ssize_t flash_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_FLASH_DATA, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t nonidx_addr_show(struct device_driver *ddp, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_nonidx_addr); -} - -static ssize_t nonidx_addr_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') - sysfs_nonidx_addr = simple_strtoul(p, &p, 16); - - return strnlen(buf, count); -} - -static ssize_t nonidx_data_show(struct device_driver *ddp, char *buf) -{ - u32 nonidx_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nonidx_data = nes_read32(nesdev->regs + sysfs_nonidx_addr); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", nonidx_data); -} - -static ssize_t nonidx_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + sysfs_nonidx_addr, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t idx_addr_show(struct device_driver *ddp, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_idx_addr); -} - -static ssize_t idx_addr_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') - sysfs_idx_addr = simple_strtoul(p, &p, 16); - - return strnlen(buf, count); -} - -static ssize_t idx_data_show(struct device_driver *ddp, char *buf) -{ - u32 idx_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - idx_data = nes_read_indexed(nesdev, sysfs_idx_addr); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", idx_data); -} - -static ssize_t idx_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write_indexed(nesdev, sysfs_idx_addr, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t wqm_quanta_show(struct device_driver *ddp, char *buf) -{ - u32 wqm_quanta_value = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - wqm_quanta_value = nesdev->nesadapter->wqm_quanta; - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%X\n", wqm_quanta_value); -} - -static ssize_t wqm_quanta_store(struct device_driver *ddp, const char *buf, - size_t count) -{ - unsigned long wqm_quanta_value; - u32 wqm_config1; - u32 i = 0; - struct nes_device *nesdev; - - if (kstrtoul(buf, 0, &wqm_quanta_value) < 0) - return -EINVAL; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nesdev->nesadapter->wqm_quanta = wqm_quanta_value; - wqm_config1 = nes_read_indexed(nesdev, - NES_IDX_WQM_CONFIG1); - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, - ((wqm_quanta_value << 1) | - (wqm_config1 & 0x00000001))); - break; - } - i++; - } - return strnlen(buf, count); -} - -static DRIVER_ATTR_RW(adapter); -static DRIVER_ATTR_RW(eeprom_cmd); -static DRIVER_ATTR_RW(eeprom_data); -static DRIVER_ATTR_RW(flash_cmd); -static DRIVER_ATTR_RW(flash_data); -static DRIVER_ATTR_RW(nonidx_addr); -static DRIVER_ATTR_RW(nonidx_data); -static DRIVER_ATTR_RW(idx_addr); -static DRIVER_ATTR_RW(idx_data); -static DRIVER_ATTR_RW(wqm_quanta); - -static struct attribute *nes_attrs[] = { - &driver_attr_adapter.attr, - &driver_attr_eeprom_cmd.attr, - &driver_attr_eeprom_data.attr, - &driver_attr_flash_cmd.attr, - &driver_attr_flash_data.attr, - &driver_attr_nonidx_addr.attr, - &driver_attr_nonidx_data.attr, - &driver_attr_idx_addr.attr, - &driver_attr_idx_data.attr, - &driver_attr_wqm_quanta.attr, - NULL, -}; -ATTRIBUTE_GROUPS(nes); - -static struct pci_driver nes_pci_driver = { - .name = DRV_NAME, - .id_table = nes_pci_table, - .probe = nes_probe, - .remove = nes_remove, - .groups = nes_groups, -}; - - -/** - * nes_init_module - module initialization entry point - */ -static int __init nes_init_module(void) -{ - int retval; - - retval = nes_cm_start(); - if (retval) { - printk(KERN_ERR PFX "Unable to start NetEffect iWARP CM.\n"); - return retval; - } - return pci_register_driver(&nes_pci_driver); -} - - -/** - * nes_exit_module - module unload entry point - */ -static void __exit nes_exit_module(void) -{ - nes_cm_stop(); - - pci_unregister_driver(&nes_pci_driver); -} - - -module_init(nes_init_module); -module_exit(nes_exit_module); diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h deleted file mode 100644 index a895fe980d10..000000000000 --- a/drivers/infiniband/hw/nes/nes.h +++ /dev/null @@ -1,574 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef __NES_H -#define __NES_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#define NES_SEND_FIRST_WRITE - -#define QUEUE_DISCONNECTS - -#define DRV_NAME "iw_nes" -#define DRV_VERSION "1.5.0.1" -#define PFX DRV_NAME ": " - -/* - * NetEffect PCI vendor id and NE010 PCI device id. - */ -#ifndef PCI_VENDOR_ID_NETEFFECT /* not in pci.ids yet */ -#define PCI_VENDOR_ID_NETEFFECT 0x1678 -#define PCI_DEVICE_ID_NETEFFECT_NE020 0x0100 -#define PCI_DEVICE_ID_NETEFFECT_NE020_KR 0x0110 -#endif - -#define NE020_REV 4 -#define NE020_REV1 5 - -#define BAR_0 0 -#define BAR_1 2 - -#define RX_BUF_SIZE (1536 + 8) -#define NES_REG0_SIZE (4 * 1024) -#define NES_TX_TIMEOUT (6*HZ) -#define NES_FIRST_QPN 64 -#define NES_SW_CONTEXT_ALIGN 1024 - -#define NES_MAX_MTU 9000 - -#define NES_NIC_MAX_NICS 16 -#define NES_MAX_ARP_TABLE_SIZE 4096 - -#define NES_NIC_CEQ_SIZE 8 -/* NICs will be on a separate CQ */ -#define NES_CCEQ_SIZE ((nesadapter->max_cq / nesadapter->port_count) - 32) - -#define NES_MAX_PORT_COUNT 4 - -#define MAX_DPC_ITERATIONS 128 - -#define NES_DRV_OPT_ENABLE_MPA_VER_0 0x00000001 -#define NES_DRV_OPT_DISABLE_MPA_CRC 0x00000002 -#define NES_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004 -#define NES_DRV_OPT_DISABLE_INTF 0x00000008 -#define NES_DRV_OPT_ENABLE_MSI 0x00000010 -#define NES_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020 -#define NES_DRV_OPT_SUPRESS_OPTION_BC 0x00000040 -#define NES_DRV_OPT_NO_INLINE_DATA 0x00000080 -#define NES_DRV_OPT_DISABLE_INT_MOD 0x00000100 -#define NES_DRV_OPT_DISABLE_VIRT_WQ 0x00000200 -#define NES_DRV_OPT_ENABLE_PAU 0x00000400 - -#define NES_AEQ_EVENT_TIMEOUT 2500 -#define NES_DISCONNECT_EVENT_TIMEOUT 2000 - -/* debug levels */ -/* must match userspace */ -#define NES_DBG_HW 0x00000001 -#define NES_DBG_INIT 0x00000002 -#define NES_DBG_ISR 0x00000004 -#define NES_DBG_PHY 0x00000008 -#define NES_DBG_NETDEV 0x00000010 -#define NES_DBG_CM 0x00000020 -#define NES_DBG_CM1 0x00000040 -#define NES_DBG_NIC_RX 0x00000080 -#define NES_DBG_NIC_TX 0x00000100 -#define NES_DBG_CQP 0x00000200 -#define NES_DBG_MMAP 0x00000400 -#define NES_DBG_MR 0x00000800 -#define NES_DBG_PD 0x00001000 -#define NES_DBG_CQ 0x00002000 -#define NES_DBG_QP 0x00004000 -#define NES_DBG_MOD_QP 0x00008000 -#define NES_DBG_AEQ 0x00010000 -#define NES_DBG_IW_RX 0x00020000 -#define NES_DBG_IW_TX 0x00040000 -#define NES_DBG_SHUTDOWN 0x00080000 -#define NES_DBG_PAU 0x00100000 -#define NES_DBG_NLMSG 0x00200000 -#define NES_DBG_RSVD1 0x10000000 -#define NES_DBG_RSVD2 0x20000000 -#define NES_DBG_RSVD3 0x40000000 -#define NES_DBG_RSVD4 0x80000000 -#define NES_DBG_ALL 0xffffffff - -#ifdef CONFIG_INFINIBAND_NES_DEBUG -#define nes_debug(level, fmt, args...) \ -do { \ - if (level & nes_debug_level) \ - printk(KERN_ERR PFX "%s[%u]: " fmt, __func__, __LINE__, ##args); \ -} while (0) - -#define NES_EVENT_TIMEOUT 1200000 -#else -#define nes_debug(level, fmt, args...) no_printk(fmt, ##args) - -#define NES_EVENT_TIMEOUT 100000 -#endif - -#include "nes_hw.h" -#include "nes_verbs.h" -#include "nes_context.h" -#include -#include "nes_cm.h" -#include "nes_mgt.h" - -extern int interrupt_mod_interval; -extern int nes_if_count; -extern int mpa_version; -extern int disable_mpa_crc; -extern unsigned int nes_drv_opt; -extern unsigned int nes_debug_level; -extern unsigned int wqm_quanta; -extern struct list_head nes_adapter_list; - -extern atomic_t cm_connects; -extern atomic_t cm_accepts; -extern atomic_t cm_disconnects; -extern atomic_t cm_closes; -extern atomic_t cm_connecteds; -extern atomic_t cm_connect_reqs; -extern atomic_t cm_rejects; -extern atomic_t mod_qp_timouts; -extern atomic_t qps_created; -extern atomic_t qps_destroyed; -extern atomic_t sw_qps_destroyed; -extern u32 mh_detected; -extern u32 mh_pauses_sent; -extern u32 cm_packets_sent; -extern u32 cm_packets_bounced; -extern u32 cm_packets_created; -extern u32 cm_packets_received; -extern u32 cm_packets_dropped; -extern u32 cm_packets_retrans; -extern atomic_t cm_listens_created; -extern atomic_t cm_listens_destroyed; -extern u32 cm_backlog_drops; -extern atomic_t cm_loopbacks; -extern atomic_t cm_nodes_created; -extern atomic_t cm_nodes_destroyed; -extern atomic_t cm_accel_dropped_pkts; -extern atomic_t cm_resets_recvd; -extern atomic_t pau_qps_created; -extern atomic_t pau_qps_destroyed; - -extern u32 int_mod_timer_init; -extern u32 int_mod_cq_depth_256; -extern u32 int_mod_cq_depth_128; -extern u32 int_mod_cq_depth_32; -extern u32 int_mod_cq_depth_24; -extern u32 int_mod_cq_depth_16; -extern u32 int_mod_cq_depth_4; -extern u32 int_mod_cq_depth_1; - -struct nes_device { - struct nes_adapter *nesadapter; - void __iomem *regs; - void __iomem *index_reg; - struct pci_dev *pcidev; - struct net_device *netdev[NES_NIC_MAX_NICS]; - u64 link_status_interrupts; - struct tasklet_struct dpc_tasklet; - spinlock_t indexed_regs_lock; - unsigned long csr_start; - unsigned long doorbell_region; - unsigned long doorbell_start; - unsigned long mac_tx_errors; - unsigned long mac_pause_frames_sent; - unsigned long mac_pause_frames_received; - unsigned long mac_rx_errors; - unsigned long mac_rx_crc_errors; - unsigned long mac_rx_symbol_err_frames; - unsigned long mac_rx_jabber_frames; - unsigned long mac_rx_oversized_frames; - unsigned long mac_rx_short_frames; - unsigned long port_rx_discards; - unsigned long port_tx_discards; - unsigned int mac_index; - unsigned int nes_stack_start; - - /* Control Structures */ - void *cqp_vbase; - dma_addr_t cqp_pbase; - u32 cqp_mem_size; - u8 ceq_index; - u8 nic_ceq_index; - struct nes_hw_cqp cqp; - struct nes_hw_cq ccq; - struct list_head cqp_avail_reqs; - struct list_head cqp_pending_reqs; - struct nes_cqp_request *nes_cqp_requests; - - u32 int_req; - u32 int_stat; - u32 timer_int_req; - u32 timer_only_int_count; - u32 intf_int_req; - u32 last_mac_tx_pauses; - u32 last_used_chunks_tx; - struct list_head list; - - u16 base_doorbell_index; - u16 currcq_count; - u16 deepcq_count; - u8 iw_status; - u8 msi_enabled; - u8 netdev_count; - u8 napi_isr_ran; - u8 disable_rx_flow_control; - u8 disable_tx_flow_control; - - struct delayed_work work; - u8 link_recheck; -}; - -/* Receive skb private area - must fit in skb->cb area */ -struct nes_rskb_cb { - u64 busaddr; - u32 maplen; - u32 seqnum; - u8 *data_start; - struct nes_qp *nesqp; -}; - -static inline __le32 get_crc_value(struct nes_v4_quad *nes_quad) -{ - u32 crc_value; - crc_value = crc32c(~0, (void *)nes_quad, sizeof (struct nes_v4_quad)); - - /* - * With commit ef19454b ("[LIB] crc32c: Keep intermediate crc - * state in cpu order"), behavior of crc32c changes on - * big-endian platforms. Our algorithm expects the previous - * behavior; otherwise we have RDMA connection establishment - * issue on big-endian. - */ - return cpu_to_le32(crc_value); -} - -static inline void -set_wqe_64bit_value(__le32 *wqe_words, u32 index, u64 value) -{ - wqe_words[index] = cpu_to_le32((u32) value); - wqe_words[index + 1] = cpu_to_le32(upper_32_bits(value)); -} - -static inline void -set_wqe_32bit_value(__le32 *wqe_words, u32 index, u32 value) -{ - wqe_words[index] = cpu_to_le32(value); -} - -static inline void -nes_fill_init_cqp_wqe(struct nes_hw_cqp_wqe *cqp_wqe, struct nes_device *nesdev) -{ - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_LEN_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_HIGH_IDX] = 0; -} - -static inline void -nes_fill_init_qp_wqe(struct nes_hw_qp_wqe *wqe, struct nes_qp *nesqp, u32 head) -{ - u32 value; - value = ((u32)((unsigned long) nesqp)) | head; - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX, - (u32)(upper_32_bits((unsigned long)(nesqp)))); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, value); -} - -/* Read from memory-mapped device */ -static inline u32 nes_read_indexed(struct nes_device *nesdev, u32 reg_index) -{ - unsigned long flags; - void __iomem *addr = nesdev->index_reg; - u32 value; - - spin_lock_irqsave(&nesdev->indexed_regs_lock, flags); - - writel(reg_index, addr); - value = readl((void __iomem *)addr + 4); - - spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags); - return value; -} - -static inline u32 nes_read32(const void __iomem *addr) -{ - return readl(addr); -} - -static inline u16 nes_read16(const void __iomem *addr) -{ - return readw(addr); -} - -static inline u8 nes_read8(const void __iomem *addr) -{ - return readb(addr); -} - -/* Write to memory-mapped device */ -static inline void nes_write_indexed(struct nes_device *nesdev, u32 reg_index, u32 val) -{ - unsigned long flags; - void __iomem *addr = nesdev->index_reg; - - spin_lock_irqsave(&nesdev->indexed_regs_lock, flags); - - writel(reg_index, addr); - writel(val, (void __iomem *)addr + 4); - - spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags); -} - -static inline void nes_write32(void __iomem *addr, u32 val) -{ - writel(val, addr); -} - -static inline void nes_write16(void __iomem *addr, u16 val) -{ - writew(val, addr); -} - -static inline void nes_write8(void __iomem *addr, u8 val) -{ - writeb(val, addr); -} - -enum nes_resource { - NES_RESOURCE_MW = 1, - NES_RESOURCE_FAST_MR, - NES_RESOURCE_PHYS_MR, - NES_RESOURCE_USER_MR, - NES_RESOURCE_PD, - NES_RESOURCE_QP, - NES_RESOURCE_CQ, - NES_RESOURCE_ARP -}; - -static inline int nes_alloc_resource(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 max_resources, - u32 *req_resource_num, u32 *next, enum nes_resource resource_type) -{ - unsigned long flags; - u32 resource_num; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - - resource_num = find_next_zero_bit(resource_array, max_resources, *next); - if (resource_num >= max_resources) { - resource_num = find_first_zero_bit(resource_array, max_resources); - if (resource_num >= max_resources) { - printk(KERN_ERR PFX "%s: No available resources [type=%u].\n", __func__, resource_type); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - return -EMFILE; - } - } - set_bit(resource_num, resource_array); - *next = resource_num+1; - if (*next == max_resources) { - *next = 0; - } - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - *req_resource_num = resource_num; - - return 0; -} - -static inline int nes_is_resource_allocated(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 resource_num) -{ - unsigned long flags; - int bit_is_set; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - - bit_is_set = test_bit(resource_num, resource_array); - nes_debug(NES_DBG_HW, "resource_num %u is%s allocated.\n", - resource_num, (bit_is_set ? "": " not")); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - - return bit_is_set; -} - -static inline void nes_free_resource(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 resource_num) -{ - unsigned long flags; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - clear_bit(resource_num, resource_array); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); -} - -static inline struct nes_vnic *to_nesvnic(struct ib_device *ibdev) -{ - return container_of(ibdev, struct nes_ib_device, ibdev)->nesvnic; -} - -static inline struct nes_pd *to_nespd(struct ib_pd *ibpd) -{ - return container_of(ibpd, struct nes_pd, ibpd); -} - -static inline struct nes_ucontext *to_nesucontext(struct ib_ucontext *ibucontext) -{ - return container_of(ibucontext, struct nes_ucontext, ibucontext); -} - -static inline struct nes_mr *to_nesmr(struct ib_mr *ibmr) -{ - return container_of(ibmr, struct nes_mr, ibmr); -} - -static inline struct nes_mr *to_nesmr_from_ibfmr(struct ib_fmr *ibfmr) -{ - return container_of(ibfmr, struct nes_mr, ibfmr); -} - -static inline struct nes_mr *to_nesmw(struct ib_mw *ibmw) -{ - return container_of(ibmw, struct nes_mr, ibmw); -} - -static inline struct nes_fmr *to_nesfmr(struct nes_mr *nesmr) -{ - return container_of(nesmr, struct nes_fmr, nesmr); -} - -static inline struct nes_cq *to_nescq(struct ib_cq *ibcq) -{ - return container_of(ibcq, struct nes_cq, ibcq); -} - -static inline struct nes_qp *to_nesqp(struct ib_qp *ibqp) -{ - return container_of(ibqp, struct nes_qp, ibqp); -} - - - -/* nes.c */ -void nes_add_ref(struct ib_qp *); -void nes_rem_ref(struct ib_qp *); -struct ib_qp *nes_get_qp(struct ib_device *, int); - - -/* nes_hw.c */ -struct nes_adapter *nes_init_adapter(struct nes_device *, u8); -void nes_nic_init_timer_defaults(struct nes_device *, u8); -void nes_destroy_adapter(struct nes_adapter *); -int nes_init_cqp(struct nes_device *); -int nes_init_phy(struct nes_device *); -int nes_init_nic_qp(struct nes_device *, struct net_device *); -void nes_destroy_nic_qp(struct nes_vnic *); -int nes_napi_isr(struct nes_device *); -void nes_dpc(unsigned long); -void nes_nic_ce_handler(struct nes_device *, struct nes_hw_nic_cq *); -void nes_iwarp_ce_handler(struct nes_device *, struct nes_hw_cq *); -int nes_destroy_cqp(struct nes_device *); -int nes_nic_cm_xmit(struct sk_buff *, struct net_device *); -void nes_recheck_link_status(struct work_struct *work); -void nes_terminate_timeout(struct timer_list *t); - -/* nes_nic.c */ -struct net_device *nes_netdev_init(struct nes_device *, void __iomem *); -void nes_netdev_destroy(struct net_device *); -int nes_nic_cm_xmit(struct sk_buff *, struct net_device *); - -/* nes_cm.c */ -void *nes_cm_create(struct net_device *); -int nes_cm_recv(struct sk_buff *, struct net_device *); -void nes_update_arp(unsigned char *, u32, u32, u16, u16); -void nes_manage_arp_cache(struct net_device *, unsigned char *, u32, u32); -void nes_sock_release(struct nes_qp *, unsigned long *); -void flush_wqes(struct nes_device *nesdev, struct nes_qp *, u32, u32); -int nes_manage_apbvt(struct nes_vnic *, u32, u32, u32); -int nes_cm_disconn(struct nes_qp *); -void nes_cm_disconn_worker(void *); - -/* nes_verbs.c */ -int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32, u32); -int nes_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); -struct nes_ib_device *nes_init_ofa_device(struct net_device *); -void nes_port_ibevent(struct nes_vnic *nesvnic); -void nes_destroy_ofa_device(struct nes_ib_device *); -int nes_register_ofa_device(struct nes_ib_device *); - -/* nes_util.c */ -int nes_read_eeprom_values(struct nes_device *, struct nes_adapter *); -void nes_write_1G_phy_reg(struct nes_device *, u8, u8, u16); -void nes_read_1G_phy_reg(struct nes_device *, u8, u8, u16 *); -void nes_write_10G_phy_reg(struct nes_device *, u16, u8, u16, u16); -void nes_read_10G_phy_reg(struct nes_device *, u8, u8, u16); -struct nes_cqp_request *nes_get_cqp_request(struct nes_device *); -void nes_free_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request); -void nes_put_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request); -void nes_post_cqp_request(struct nes_device *, struct nes_cqp_request *); -int nes_arp_table(struct nes_device *, u32, u8 *, u32); -void nes_mh_fix(struct timer_list *t); -void nes_clc(struct timer_list *t); -void nes_dump_mem(unsigned int, void *, int); -u32 nes_crc32(u32, u32, u32, u32, u8 *, u32, u32, u32); - -#endif /* __NES_H */ diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c deleted file mode 100644 index 62bf986eba67..000000000000 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ /dev/null @@ -1,3992 +0,0 @@ -/* - * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - - -#define TCPOPT_TIMESTAMP 8 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "nes.h" - -u32 cm_packets_sent; -u32 cm_packets_bounced; -u32 cm_packets_dropped; -u32 cm_packets_retrans; -u32 cm_packets_created; -u32 cm_packets_received; -atomic_t cm_listens_created; -atomic_t cm_listens_destroyed; -u32 cm_backlog_drops; -atomic_t cm_loopbacks; -atomic_t cm_nodes_created; -atomic_t cm_nodes_destroyed; -atomic_t cm_accel_dropped_pkts; -atomic_t cm_resets_recvd; - -static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *); -static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *); -static int mini_cm_del_listen(struct nes_cm_core *, struct nes_cm_listener *); -static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *, struct nes_vnic *, u16, void *, struct nes_cm_info *); -static int mini_cm_close(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_accept(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_reject(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_recv_pkt(struct nes_cm_core *, struct nes_vnic *, struct sk_buff *); -static int mini_cm_dealloc_core(struct nes_cm_core *); -static int mini_cm_get(struct nes_cm_core *); -static int mini_cm_set(struct nes_cm_core *, u32, u32); - -static void form_cm_frame(struct sk_buff *, struct nes_cm_node *, void *, u32, void *, u32, u8); -static int add_ref_cm_node(struct nes_cm_node *); -static int rem_ref_cm_node(struct nes_cm_core *, struct nes_cm_node *); - -static int nes_cm_disconn_true(struct nes_qp *); -static int nes_cm_post_event(struct nes_cm_event *event); -static int nes_disconnect(struct nes_qp *nesqp, int abrupt); -static void nes_disconnect_worker(struct work_struct *work); - -static int send_mpa_request(struct nes_cm_node *, struct sk_buff *); -static int send_mpa_reject(struct nes_cm_node *); -static int send_syn(struct nes_cm_node *, u32, struct sk_buff *); -static int send_reset(struct nes_cm_node *, struct sk_buff *); -static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb); -static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb); -static void process_packet(struct nes_cm_node *, struct sk_buff *, struct nes_cm_core *); - -static void active_open_err(struct nes_cm_node *, struct sk_buff *, int); -static void passive_open_err(struct nes_cm_node *, struct sk_buff *, int); -static void cleanup_retrans_entry(struct nes_cm_node *); -static void handle_rcv_mpa(struct nes_cm_node *, struct sk_buff *); -static void free_retrans_entry(struct nes_cm_node *cm_node); -static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, struct sk_buff *skb, int optionsize, int passive); - -/* CM event handler functions */ -static void cm_event_connected(struct nes_cm_event *); -static void cm_event_connect_error(struct nes_cm_event *); -static void cm_event_reset(struct nes_cm_event *); -static void cm_event_mpa_req(struct nes_cm_event *); -static void cm_event_mpa_reject(struct nes_cm_event *); -static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node); - -/* MPA build functions */ -static int cm_build_mpa_frame(struct nes_cm_node *, u8 **, u16 *, u8 *, u8); -static void build_mpa_v2(struct nes_cm_node *, void *, u8); -static void build_mpa_v1(struct nes_cm_node *, void *, u8); -static void build_rdma0_msg(struct nes_cm_node *, struct nes_qp **); - -static void print_core(struct nes_cm_core *core); -static void record_ird_ord(struct nes_cm_node *, u16, u16); - -/* External CM API Interface */ -/* instance of function pointers for client API */ -/* set address of this instance to cm_core->cm_ops at cm_core alloc */ -static const struct nes_cm_ops nes_cm_api = { - .accelerated = mini_cm_accelerated, - .listen = mini_cm_listen, - .stop_listener = mini_cm_del_listen, - .connect = mini_cm_connect, - .close = mini_cm_close, - .accept = mini_cm_accept, - .reject = mini_cm_reject, - .recv_pkt = mini_cm_recv_pkt, - .destroy_cm_core = mini_cm_dealloc_core, - .get = mini_cm_get, - .set = mini_cm_set -}; - -static struct nes_cm_core *g_cm_core; - -atomic_t cm_connects; -atomic_t cm_accepts; -atomic_t cm_disconnects; -atomic_t cm_closes; -atomic_t cm_connecteds; -atomic_t cm_connect_reqs; -atomic_t cm_rejects; - -int nes_add_ref_cm_node(struct nes_cm_node *cm_node) -{ - return add_ref_cm_node(cm_node); -} - -int nes_rem_ref_cm_node(struct nes_cm_node *cm_node) -{ - return rem_ref_cm_node(cm_node->cm_core, cm_node); -} -/** - * create_event - */ -static struct nes_cm_event *create_event(struct nes_cm_node * cm_node, - enum nes_cm_event_type type) -{ - struct nes_cm_event *event; - - if (!cm_node->cm_id) - return NULL; - - /* allocate an empty event */ - event = kzalloc(sizeof(*event), GFP_ATOMIC); - - if (!event) - return NULL; - - event->type = type; - event->cm_node = cm_node; - event->cm_info.rem_addr = cm_node->rem_addr; - event->cm_info.loc_addr = cm_node->loc_addr; - event->cm_info.rem_port = cm_node->rem_port; - event->cm_info.loc_port = cm_node->loc_port; - event->cm_info.cm_id = cm_node->cm_id; - - nes_debug(NES_DBG_CM, "cm_node=%p Created event=%p, type=%u, " - "dst_addr=%08x[%x], src_addr=%08x[%x]\n", - cm_node, event, type, event->cm_info.loc_addr, - event->cm_info.loc_port, event->cm_info.rem_addr, - event->cm_info.rem_port); - - nes_cm_post_event(event); - return event; -} - - -/** - * send_mpa_request - */ -static int send_mpa_request(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - - if (!skb) { - nes_debug(NES_DBG_CM, "skb set to NULL\n"); - return -1; - } - - /* send an MPA Request frame */ - cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REQUEST); - form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK); - - return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); -} - - - -static int send_mpa_reject(struct nes_cm_node *cm_node) -{ - struct sk_buff *skb = NULL; - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - struct ietf_mpa_v1 *mpa_frame; - - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -ENOMEM; - } - - /* send an MPA reject frame */ - cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REPLY); - mpa_frame = (struct ietf_mpa_v1 *)*start_buff; - mpa_frame->flags |= IETF_MPA_FLAGS_REJECT; - form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK | SET_FIN); - - cm_node->state = NES_CM_STATE_FIN_WAIT1; - return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); -} - - -/** - * recv_mpa - process a received TCP pkt, we are expecting an - * IETF MPA frame - */ -static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type, - u32 len) -{ - struct ietf_mpa_v1 *mpa_frame; - struct ietf_mpa_v2 *mpa_v2_frame; - struct ietf_rtr_msg *rtr_msg; - int mpa_hdr_len; - int priv_data_len; - - *type = NES_MPA_REQUEST_ACCEPT; - - /* assume req frame is in tcp data payload */ - if (len < sizeof(struct ietf_mpa_v1)) { - nes_debug(NES_DBG_CM, "The received ietf buffer was too small (%x)\n", len); - return -EINVAL; - } - - /* points to the beginning of the frame, which could be MPA V1 or V2 */ - mpa_frame = (struct ietf_mpa_v1 *)buffer; - mpa_hdr_len = sizeof(struct ietf_mpa_v1); - priv_data_len = ntohs(mpa_frame->priv_data_len); - - /* make sure mpa private data len is less than 512 bytes */ - if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { - nes_debug(NES_DBG_CM, "The received Length of Private" - " Data field exceeds 512 octets\n"); - return -EINVAL; - } - /* - * make sure MPA receiver interoperate with the - * received MPA version and MPA key information - * - */ - if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { - nes_debug(NES_DBG_CM, "The received mpa version" - " is not supported\n"); - return -EINVAL; - } - /* - * backwards compatibility only - */ - if (mpa_frame->rev > cm_node->mpa_frame_rev) { - nes_debug(NES_DBG_CM, "The received mpa version" - " can not be interoperated\n"); - return -EINVAL; - } else { - cm_node->mpa_frame_rev = mpa_frame->rev; - } - - if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { - nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n"); - return -EINVAL; - } - } else { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { - nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n"); - return -EINVAL; - } - } - - if (priv_data_len + mpa_hdr_len != len) { - nes_debug(NES_DBG_CM, "The received ietf buffer was not right" - " complete (%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); - return -EINVAL; - } - /* make sure it does not exceed the max size */ - if (len > MAX_CM_BUFFER) { - nes_debug(NES_DBG_CM, "The received ietf buffer was too large" - " (%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); - return -EINVAL; - } - - cm_node->mpa_frame_size = priv_data_len; - - switch (mpa_frame->rev) { - case IETF_MPA_V2: { - u16 ird_size; - u16 ord_size; - u16 rtr_ctrl_ird; - u16 rtr_ctrl_ord; - - mpa_v2_frame = (struct ietf_mpa_v2 *)buffer; - mpa_hdr_len += IETF_RTR_MSG_SIZE; - cm_node->mpa_frame_size -= IETF_RTR_MSG_SIZE; - rtr_msg = &mpa_v2_frame->rtr_msg; - - /* parse rtr message */ - rtr_ctrl_ird = ntohs(rtr_msg->ctrl_ird); - rtr_ctrl_ord = ntohs(rtr_msg->ctrl_ord); - ird_size = rtr_ctrl_ird & IETF_NO_IRD_ORD; - ord_size = rtr_ctrl_ord & IETF_NO_IRD_ORD; - - if (!(rtr_ctrl_ird & IETF_PEER_TO_PEER)) { - /* send reset */ - return -EINVAL; - } - if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) - cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; - - if (cm_node->mpav2_ird_ord != IETF_NO_IRD_ORD) { - /* responder */ - if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { - /* we are still negotiating */ - if (ord_size > NES_MAX_IRD) { - cm_node->ird_size = NES_MAX_IRD; - } else { - cm_node->ird_size = ord_size; - if (ord_size == 0 && - (rtr_ctrl_ord & IETF_RDMA0_READ)) { - cm_node->ird_size = 1; - nes_debug(NES_DBG_CM, - "%s: Remote peer doesn't support RDMA0_READ (ord=%u)\n", - __func__, ord_size); - } - } - if (ird_size > NES_MAX_ORD) - cm_node->ord_size = NES_MAX_ORD; - else - cm_node->ord_size = ird_size; - } else { /* initiator */ - if (ord_size > NES_MAX_IRD) { - nes_debug(NES_DBG_CM, - "%s: Unable to support the requested (ord =%u)\n", - __func__, ord_size); - return -EINVAL; - } - cm_node->ird_size = ord_size; - - if (ird_size > NES_MAX_ORD) { - cm_node->ord_size = NES_MAX_ORD; - } else { - if (ird_size == 0 && - (rtr_ctrl_ord & IETF_RDMA0_READ)) { - nes_debug(NES_DBG_CM, - "%s: Remote peer doesn't support RDMA0_READ (ird=%u)\n", - __func__, ird_size); - return -EINVAL; - } else { - cm_node->ord_size = ird_size; - } - } - } - } - - if (rtr_ctrl_ord & IETF_RDMA0_READ) { - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - - } else if (rtr_ctrl_ord & IETF_RDMA0_WRITE) { - cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; - } else { /* Not supported RDMA0 operation */ - return -EINVAL; - } - break; - } - case IETF_MPA_V1: - default: - break; - } - - /* copy entire MPA frame to our cm_node's frame */ - memcpy(cm_node->mpa_frame_buf, buffer + mpa_hdr_len, cm_node->mpa_frame_size); - - if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) - *type = NES_MPA_REQUEST_REJECT; - return 0; -} - - -/** - * form_cm_frame - get a free packet and build empty frame Use - * node info to build. - */ -static void form_cm_frame(struct sk_buff *skb, - struct nes_cm_node *cm_node, void *options, u32 optionsize, - void *data, u32 datasize, u8 flags) -{ - struct tcphdr *tcph; - struct iphdr *iph; - struct ethhdr *ethh; - u8 *buf; - u16 packetsize = sizeof(*iph); - - packetsize += sizeof(*tcph); - packetsize += optionsize + datasize; - - skb_trim(skb, 0); - memset(skb->data, 0x00, ETH_HLEN + sizeof(*iph) + sizeof(*tcph)); - - buf = skb_put(skb, packetsize + ETH_HLEN); - - ethh = (struct ethhdr *)buf; - buf += ETH_HLEN; - - iph = (struct iphdr *)buf; - buf += sizeof(*iph); - tcph = (struct tcphdr *)buf; - skb_reset_mac_header(skb); - skb_set_network_header(skb, ETH_HLEN); - skb_set_transport_header(skb, ETH_HLEN + sizeof(*iph)); - buf += sizeof(*tcph); - - skb->ip_summed = CHECKSUM_PARTIAL; - if (!(cm_node->netdev->features & NETIF_F_IP_CSUM)) - skb->ip_summed = CHECKSUM_NONE; - skb->protocol = htons(0x800); - skb->data_len = 0; - skb->mac_len = ETH_HLEN; - - memcpy(ethh->h_dest, cm_node->rem_mac, ETH_ALEN); - memcpy(ethh->h_source, cm_node->loc_mac, ETH_ALEN); - ethh->h_proto = htons(0x0800); - - iph->version = IPVERSION; - iph->ihl = 5; /* 5 * 4Byte words, IP headr len */ - iph->tos = 0; - iph->tot_len = htons(packetsize); - iph->id = htons(++cm_node->tcp_cntxt.loc_id); - - iph->frag_off = htons(0x4000); - iph->ttl = 0x40; - iph->protocol = 0x06; /* IPPROTO_TCP */ - - iph->saddr = htonl(cm_node->loc_addr); - iph->daddr = htonl(cm_node->rem_addr); - - tcph->source = htons(cm_node->loc_port); - tcph->dest = htons(cm_node->rem_port); - tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num); - - if (flags & SET_ACK) { - cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; - tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num); - tcph->ack = 1; - } else { - tcph->ack_seq = 0; - } - - if (flags & SET_SYN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->syn = 1; - } else { - cm_node->tcp_cntxt.loc_seq_num += datasize; - } - - if (flags & SET_FIN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->fin = 1; - } - - if (flags & SET_RST) - tcph->rst = 1; - - tcph->doff = (u16)((sizeof(*tcph) + optionsize + 3) >> 2); - tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd); - tcph->urg_ptr = 0; - if (optionsize) - memcpy(buf, options, optionsize); - buf += optionsize; - if (datasize) - memcpy(buf, data, datasize); - - skb_shinfo(skb)->nr_frags = 0; - cm_packets_created++; -} - -/** - * print_core - dump a cm core - */ -static void print_core(struct nes_cm_core *core) -{ - nes_debug(NES_DBG_CM, "---------------------------------------------\n"); - nes_debug(NES_DBG_CM, "CM Core -- (core = %p )\n", core); - if (!core) - return; - nes_debug(NES_DBG_CM, "---------------------------------------------\n"); - - nes_debug(NES_DBG_CM, "State : %u \n", core->state); - - nes_debug(NES_DBG_CM, "Listen Nodes : %u \n", atomic_read(&core->listen_node_cnt)); - nes_debug(NES_DBG_CM, "Active Nodes : %u \n", atomic_read(&core->node_cnt)); - - nes_debug(NES_DBG_CM, "core : %p \n", core); - - nes_debug(NES_DBG_CM, "-------------- end core ---------------\n"); -} - -static void record_ird_ord(struct nes_cm_node *cm_node, - u16 conn_ird, u16 conn_ord) -{ - if (conn_ird > NES_MAX_IRD) - conn_ird = NES_MAX_IRD; - - if (conn_ord > NES_MAX_ORD) - conn_ord = NES_MAX_ORD; - - cm_node->ird_size = conn_ird; - cm_node->ord_size = conn_ord; -} - -/** - * cm_build_mpa_frame - build a MPA V1 frame or MPA V2 frame - */ -static int cm_build_mpa_frame(struct nes_cm_node *cm_node, u8 **start_buff, - u16 *buff_len, u8 *pci_mem, u8 mpa_key) -{ - int ret = 0; - - *start_buff = (pci_mem) ? pci_mem : &cm_node->mpa_frame_buf[0]; - - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V1: - *start_buff = (u8 *)*start_buff + sizeof(struct ietf_rtr_msg); - *buff_len = sizeof(struct ietf_mpa_v1) + cm_node->mpa_frame_size; - build_mpa_v1(cm_node, *start_buff, mpa_key); - break; - case IETF_MPA_V2: - *buff_len = sizeof(struct ietf_mpa_v2) + cm_node->mpa_frame_size; - build_mpa_v2(cm_node, *start_buff, mpa_key); - break; - default: - ret = -EINVAL; - } - return ret; -} - -/** - * build_mpa_v2 - build a MPA V2 frame - */ -static void build_mpa_v2(struct nes_cm_node *cm_node, - void *start_addr, u8 mpa_key) -{ - struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr; - struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; - u16 ctrl_ird; - u16 ctrl_ord; - - /* initialize the upper 5 bytes of the frame */ - build_mpa_v1(cm_node, start_addr, mpa_key); - mpa_frame->flags |= IETF_MPA_V2_FLAG; /* set a bit to indicate MPA V2 */ - mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE); - - /* initialize RTR msg */ - if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - ctrl_ird = IETF_NO_IRD_ORD; - ctrl_ord = IETF_NO_IRD_ORD; - } else { - ctrl_ird = cm_node->ird_size & IETF_NO_IRD_ORD; - ctrl_ord = cm_node->ord_size & IETF_NO_IRD_ORD; - } - ctrl_ird |= IETF_PEER_TO_PEER; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - ctrl_ord |= IETF_RDMA0_WRITE; - ctrl_ord |= IETF_RDMA0_READ; - break; - case MPA_KEY_REPLY: - switch (cm_node->send_rdma0_op) { - case SEND_RDMA_WRITE_ZERO: - ctrl_ord |= IETF_RDMA0_WRITE; - break; - case SEND_RDMA_READ_ZERO: - ctrl_ord |= IETF_RDMA0_READ; - break; - } - } - rtr_msg->ctrl_ird = htons(ctrl_ird); - rtr_msg->ctrl_ord = htons(ctrl_ord); -} - -/** - * build_mpa_v1 - build a MPA V1 frame - */ -static void build_mpa_v1(struct nes_cm_node *cm_node, void *start_addr, u8 mpa_key) -{ - struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); - break; - case MPA_KEY_REPLY: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); - break; - } - mpa_frame->flags = IETF_MPA_FLAGS_CRC; - mpa_frame->rev = cm_node->mpa_frame_rev; - mpa_frame->priv_data_len = htons(cm_node->mpa_frame_size); -} - -static void build_rdma0_msg(struct nes_cm_node *cm_node, struct nes_qp **nesqp_addr) -{ - u64 u64temp; - struct nes_qp *nesqp = *nesqp_addr; - struct nes_hw_qp_wqe *wqe = &nesqp->hwqp.sq_vbase[0]; - - u64temp = (unsigned long)nesqp->nesuqp_addr; - u64temp |= NES_SW_CONTEXT_ALIGN >> 1; - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, u64temp); - - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = 0; - - switch (cm_node->send_rdma0_op) { - case SEND_RDMA_WRITE_ZERO: - nes_debug(NES_DBG_CM, "Sending first write.\n"); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_OP_RDMAW); - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 0; - break; - - case SEND_RDMA_READ_ZERO: - default: - if (cm_node->send_rdma0_op != SEND_RDMA_READ_ZERO) - WARN(1, "Unsupported RDMA0 len operation=%u\n", - cm_node->send_rdma0_op); - nes_debug(NES_DBG_CM, "Sending first rdma operation.\n"); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_OP_RDMAR); - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = 1; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = 1; - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 1; - break; - } - - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - - /*use the reserved spot on the WQ for the extra first WQE*/ - nesqp->nesqp_context->ird_ord_sizes &= cpu_to_le32(~(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT | - NES_QPCONTEXT_ORDIRD_WRPDU | - NES_QPCONTEXT_ORDIRD_ALSMM)); - nesqp->skip_lsmm = 1; - nesqp->hwqp.sq_tail = 0; -} - -/** - * schedule_nes_timer - * note - cm_node needs to be protected before calling this. Encase in: - * rem_ref_cm_node(cm_core, cm_node);add_ref_cm_node(cm_node); - */ -int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb, - enum nes_timer_type type, int send_retrans, - int close_when_complete) -{ - unsigned long flags; - struct nes_cm_core *cm_core = cm_node->cm_core; - struct nes_timer_entry *new_send; - int ret = 0; - - new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); - if (!new_send) - return -ENOMEM; - - /* new_send->timetosend = currenttime */ - new_send->retrycount = NES_DEFAULT_RETRYS; - new_send->retranscount = NES_DEFAULT_RETRANS; - new_send->skb = skb; - new_send->timetosend = jiffies; - new_send->type = type; - new_send->netdev = cm_node->netdev; - new_send->send_retrans = send_retrans; - new_send->close_when_complete = close_when_complete; - - if (type == NES_TIMER_TYPE_CLOSE) { - new_send->timetosend += (HZ / 10); - if (cm_node->recv_entry) { - kfree(new_send); - WARN_ON(1); - return -EINVAL; - } - cm_node->recv_entry = new_send; - } - - if (type == NES_TIMER_TYPE_SEND) { - new_send->seq_num = ntohl(tcp_hdr(skb)->seq); - refcount_inc(&new_send->skb->users); - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - cm_node->send_entry = new_send; - add_ref_cm_node(cm_node); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - new_send->timetosend = jiffies + NES_RETRY_TIMEOUT; - - ret = nes_nic_cm_xmit(new_send->skb, cm_node->netdev); - if (ret != NETDEV_TX_OK) { - nes_debug(NES_DBG_CM, "Error sending packet %p " - "(jiffies = %lu)\n", new_send, jiffies); - new_send->timetosend = jiffies; - ret = NETDEV_TX_OK; - } else { - cm_packets_sent++; - if (!send_retrans) { - cleanup_retrans_entry(cm_node); - if (close_when_complete) - rem_ref_cm_node(cm_core, cm_node); - return ret; - } - } - } - - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, new_send->timetosend); - - return ret; -} - -static void nes_retrans_expired(struct nes_cm_node *cm_node) -{ - struct iw_cm_id *cm_id = cm_node->cm_id; - enum nes_cm_node_state state = cm_node->state; - cm_node->state = NES_CM_STATE_CLOSED; - - switch (state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_CLOSING: - rem_ref_cm_node(cm_node->cm_core, cm_node); - break; - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_FIN_WAIT1: - if (cm_node->cm_id) - cm_id->rem_ref(cm_id); - send_reset(cm_node, NULL); - break; - default: - add_ref_cm_node(cm_node); - send_reset(cm_node, NULL); - create_event(cm_node, NES_CM_EVENT_ABORTED); - } -} - -static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node) -{ - struct nes_timer_entry *recv_entry = cm_node->recv_entry; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct nes_qp *nesqp; - unsigned long qplockflags; - - if (!recv_entry) - return; - nesqp = (struct nes_qp *)recv_entry->skb; - if (nesqp) { - spin_lock_irqsave(&nesqp->lock, qplockflags); - if (nesqp->cm_id) { - nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, " - "refcount = %d: HIT A " - "NES_TIMER_TYPE_CLOSE with something " - "to do!!!\n", nesqp->hwqp.qp_id, cm_id, - atomic_read(&nesqp->refcount)); - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; - nesqp->ibqp_state = IB_QPS_ERR; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_cm_disconn(nesqp); - } else { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, " - "refcount = %d: HIT A " - "NES_TIMER_TYPE_CLOSE with nothing " - "to do!!!\n", nesqp->hwqp.qp_id, cm_id, - atomic_read(&nesqp->refcount)); - } - } else if (rem_node) { - /* TIME_WAIT state */ - rem_ref_cm_node(cm_node->cm_core, cm_node); - } - if (cm_node->cm_id) - cm_id->rem_ref(cm_id); - kfree(recv_entry); - cm_node->recv_entry = NULL; -} - -/** - * nes_cm_timer_tick - */ -static void nes_cm_timer_tick(struct timer_list *unused) -{ - unsigned long flags; - unsigned long nexttimeout = jiffies + NES_LONG_TIME; - struct nes_cm_node *cm_node; - struct nes_timer_entry *send_entry, *recv_entry; - struct list_head *list_core_temp; - struct list_head *list_node; - struct nes_cm_core *cm_core = g_cm_core; - u32 settimer = 0; - unsigned long timetosend; - int ret = NETDEV_TX_OK; - - struct list_head timer_list; - - INIT_LIST_HEAD(&timer_list); - spin_lock_irqsave(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, - &cm_core->connected_nodes) { - cm_node = container_of(list_node, struct nes_cm_node, list); - if ((cm_node->recv_entry) || (cm_node->send_entry)) { - add_ref_cm_node(cm_node); - list_add(&cm_node->timer_entry, &timer_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, &timer_list) { - cm_node = container_of(list_node, struct nes_cm_node, - timer_entry); - recv_entry = cm_node->recv_entry; - - if (recv_entry) { - if (time_after(recv_entry->timetosend, jiffies)) { - if (nexttimeout > recv_entry->timetosend || - !settimer) { - nexttimeout = recv_entry->timetosend; - settimer = 1; - } - } else { - handle_recv_entry(cm_node, 1); - } - } - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - do { - send_entry = cm_node->send_entry; - if (!send_entry) - break; - if (time_after(send_entry->timetosend, jiffies)) { - if (cm_node->state != NES_CM_STATE_TSA) { - if ((nexttimeout > - send_entry->timetosend) || - !settimer) { - nexttimeout = - send_entry->timetosend; - settimer = 1; - } - } else { - free_retrans_entry(cm_node); - } - break; - } - - if ((cm_node->state == NES_CM_STATE_TSA) || - (cm_node->state == NES_CM_STATE_CLOSED)) { - free_retrans_entry(cm_node); - break; - } - - if (!send_entry->retranscount || - !send_entry->retrycount) { - cm_packets_dropped++; - free_retrans_entry(cm_node); - - spin_unlock_irqrestore( - &cm_node->retrans_list_lock, flags); - nes_retrans_expired(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - spin_lock_irqsave(&cm_node->retrans_list_lock, - flags); - break; - } - refcount_inc(&send_entry->skb->users); - cm_packets_retrans++; - nes_debug(NES_DBG_CM, "Retransmitting send_entry %p " - "for node %p, jiffies = %lu, time to send = " - "%lu, retranscount = %u, send_entry->seq_num = " - "0x%08X, cm_node->tcp_cntxt.rem_ack_num = " - "0x%08X\n", send_entry, cm_node, jiffies, - send_entry->timetosend, - send_entry->retranscount, - send_entry->seq_num, - cm_node->tcp_cntxt.rem_ack_num); - - spin_unlock_irqrestore(&cm_node->retrans_list_lock, - flags); - ret = nes_nic_cm_xmit(send_entry->skb, cm_node->netdev); - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - if (ret != NETDEV_TX_OK) { - nes_debug(NES_DBG_CM, "rexmit failed for " - "node=%p\n", cm_node); - cm_packets_bounced++; - send_entry->retrycount--; - nexttimeout = jiffies + NES_SHORT_TIME; - settimer = 1; - break; - } else { - cm_packets_sent++; - } - nes_debug(NES_DBG_CM, "Packet Sent: retrans count = " - "%u, retry count = %u.\n", - send_entry->retranscount, - send_entry->retrycount); - if (send_entry->send_retrans) { - send_entry->retranscount--; - timetosend = (NES_RETRY_TIMEOUT << - (NES_DEFAULT_RETRANS - send_entry->retranscount)); - - send_entry->timetosend = jiffies + - min(timetosend, NES_MAX_TIMEOUT); - if (nexttimeout > send_entry->timetosend || - !settimer) { - nexttimeout = send_entry->timetosend; - settimer = 1; - } - } else { - int close_when_complete; - close_when_complete = - send_entry->close_when_complete; - nes_debug(NES_DBG_CM, "cm_node=%p state=%d\n", - cm_node, cm_node->state); - free_retrans_entry(cm_node); - if (close_when_complete) - rem_ref_cm_node(cm_node->cm_core, - cm_node); - } - } while (0); - - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } - - if (settimer) { - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, nexttimeout); - } -} - - -/** - * send_syn - */ -static int send_syn(struct nes_cm_node *cm_node, u32 sendack, - struct sk_buff *skb) -{ - int ret; - int flags = SET_SYN; - char optionsbuffer[sizeof(struct option_mss) + - sizeof(struct option_windowscale) + sizeof(struct option_base) + - TCP_OPTIONS_PADDING]; - - int optionssize = 0; - /* Sending MSS option */ - union all_known_options *options; - - if (!cm_node) - return -EINVAL; - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_mss.optionnum = OPTION_NUMBER_MSS; - options->as_mss.length = sizeof(struct option_mss); - options->as_mss.mss = htons(cm_node->tcp_cntxt.mss); - optionssize += sizeof(struct option_mss); - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE; - options->as_windowscale.length = sizeof(struct option_windowscale); - options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; - optionssize += sizeof(struct option_windowscale); - - if (sendack && !(NES_DRV_OPT_SUPRESS_OPTION_BC & nes_drv_opt)) { - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_base.optionnum = OPTION_NUMBER_WRITE0; - options->as_base.length = sizeof(struct option_base); - optionssize += sizeof(struct option_base); - /* we need the size to be a multiple of 4 */ - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = 1; - optionssize += 1; - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = 1; - optionssize += 1; - } - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = OPTION_NUMBER_END; - optionssize += 1; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - if (sendack) - flags |= SET_ACK; - - form_cm_frame(skb, cm_node, optionsbuffer, optionssize, NULL, 0, flags); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); - - return ret; -} - - -/** - * send_reset - */ -static int send_reset(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - int flags = SET_RST | SET_ACK; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -ENOMEM; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, flags); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 1); - - return ret; -} - - -/** - * send_ack - */ -static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 0); - - return ret; -} - - -/** - * send_fin - */ -static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - - /* if we didn't get a frame get one */ - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK | SET_FIN); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); - - return ret; -} - - -/** - * find_node - find a cm node that matches the reference cm node - */ -static struct nes_cm_node *find_node(struct nes_cm_core *cm_core, - u16 rem_port, nes_addr_t rem_addr, u16 loc_port, nes_addr_t loc_addr) -{ - unsigned long flags; - struct list_head *hte; - struct nes_cm_node *cm_node; - - /* get a handle on the hte */ - hte = &cm_core->connected_nodes; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_entry(cm_node, hte, list) { - /* compare quad, return node handle if a match */ - nes_debug(NES_DBG_CM, "finding node %x:%x =? %x:%x ^ %x:%x =? %x:%x\n", - cm_node->loc_addr, cm_node->loc_port, - loc_addr, loc_port, - cm_node->rem_addr, cm_node->rem_port, - rem_addr, rem_port); - if ((cm_node->loc_addr == loc_addr) && - (cm_node->loc_port == loc_port) && - (cm_node->rem_addr == rem_addr) && - (cm_node->rem_port == rem_port)) { - add_ref_cm_node(cm_node); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - return cm_node; - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - /* no owner node */ - return NULL; -} - - -/** - * find_listener - find a cm node listening on this addr-port pair - */ -static struct nes_cm_listener *find_listener(struct nes_cm_core *cm_core, - nes_addr_t dst_addr, u16 dst_port, - enum nes_cm_listener_state listener_state) -{ - unsigned long flags; - struct nes_cm_listener *listen_node; - nes_addr_t listen_addr; - u16 listen_port; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_for_each_entry(listen_node, &cm_core->listen_list.list, list) { - listen_addr = listen_node->loc_addr; - listen_port = listen_node->loc_port; - - /* compare node pair, return node handle if a match */ - if (((listen_addr == dst_addr) || - listen_addr == 0x00000000) && - (listen_port == dst_port) && - (listener_state & listen_node->listener_state)) { - atomic_inc(&listen_node->ref_count); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - return listen_node; - } - } - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - /* no listener */ - return NULL; -} - -/** - * add_hte_node - add a cm node to the hash table - */ -static int add_hte_node(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - unsigned long flags; - struct list_head *hte; - - if (!cm_node || !cm_core) - return -EINVAL; - - nes_debug(NES_DBG_CM, "Adding Node %p to Active Connection HT\n", - cm_node); - - spin_lock_irqsave(&cm_core->ht_lock, flags); - - /* get a handle on the hash table element (list head for this slot) */ - hte = &cm_core->connected_nodes; - list_add_tail(&cm_node->list, hte); - atomic_inc(&cm_core->ht_node_cnt); - - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - return 0; -} - - -/** - * mini_cm_dec_refcnt_listen - */ -static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core, - struct nes_cm_listener *listener, int free_hanging_nodes) -{ - int ret = -EINVAL; - int err = 0; - unsigned long flags; - struct list_head *list_pos = NULL; - struct list_head *list_temp = NULL; - struct nes_cm_node *cm_node = NULL; - struct list_head reset_list; - - nes_debug(NES_DBG_CM, "attempting listener= %p free_nodes= %d, " - "refcnt=%d\n", listener, free_hanging_nodes, - atomic_read(&listener->ref_count)); - /* free non-accelerated child nodes for this listener */ - INIT_LIST_HEAD(&reset_list); - if (free_hanging_nodes) { - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_safe(list_pos, list_temp, - &g_cm_core->connected_nodes) { - cm_node = container_of(list_pos, struct nes_cm_node, - list); - if ((cm_node->listener == listener) && - (!cm_node->accelerated)) { - add_ref_cm_node(cm_node); - list_add(&cm_node->reset_entry, &reset_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - } - - list_for_each_safe(list_pos, list_temp, &reset_list) { - cm_node = container_of(list_pos, struct nes_cm_node, - reset_entry); - { - struct nes_cm_node *loopback = cm_node->loopbackpartner; - enum nes_cm_node_state old_state; - if (NES_CM_STATE_FIN_WAIT1 <= cm_node->state) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - } else { - if (!loopback) { - cleanup_retrans_entry(cm_node); - err = send_reset(cm_node, NULL); - if (err) { - cm_node->state = - NES_CM_STATE_CLOSED; - WARN_ON(1); - } else { - old_state = cm_node->state; - cm_node->state = NES_CM_STATE_LISTENER_DESTROYED; - if (old_state != NES_CM_STATE_MPAREQ_RCVD) - rem_ref_cm_node( - cm_node->cm_core, - cm_node); - } - } else { - struct nes_cm_event event; - - event.cm_node = loopback; - event.cm_info.rem_addr = - loopback->rem_addr; - event.cm_info.loc_addr = - loopback->loc_addr; - event.cm_info.rem_port = - loopback->rem_port; - event.cm_info.loc_port = - loopback->loc_port; - event.cm_info.cm_id = loopback->cm_id; - add_ref_cm_node(loopback); - loopback->state = NES_CM_STATE_CLOSED; - cm_event_connect_error(&event); - cm_node->state = NES_CM_STATE_LISTENER_DESTROYED; - - rem_ref_cm_node(cm_node->cm_core, - cm_node); - - } - } - } - } - - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - if (!atomic_dec_return(&listener->ref_count)) { - list_del(&listener->list); - - /* decrement our listen node count */ - atomic_dec(&cm_core->listen_node_cnt); - - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - if (listener->nesvnic) { - nes_manage_apbvt(listener->nesvnic, - listener->loc_port, - PCI_FUNC(listener->nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - - nes_debug(NES_DBG_NLMSG, - "Delete APBVT loc_port = %04X\n", - listener->loc_port); - } - - nes_debug(NES_DBG_CM, "destroying listener (%p)\n", listener); - - kfree(listener); - listener = NULL; - ret = 0; - atomic_inc(&cm_listens_destroyed); - } else { - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - } - if (listener) { - if (atomic_read(&listener->pend_accepts_cnt) > 0) - nes_debug(NES_DBG_CM, "destroying listener (%p)" - " with non-zero pending accepts=%u\n", - listener, atomic_read(&listener->pend_accepts_cnt)); - } - - return ret; -} - - -/** - * mini_cm_del_listen - */ -static int mini_cm_del_listen(struct nes_cm_core *cm_core, - struct nes_cm_listener *listener) -{ - listener->listener_state = NES_CM_LISTENER_PASSIVE_STATE; - listener->cm_id = NULL; /* going to be destroyed pretty soon */ - return mini_cm_dec_refcnt_listen(cm_core, listener, 1); -} - - -/** - * mini_cm_accelerated - */ -static inline int mini_cm_accelerated(struct nes_cm_core *cm_core, - struct nes_cm_node *cm_node) -{ - cm_node->accelerated = true; - - if (cm_node->accept_pend) { - BUG_ON(!cm_node->listener); - atomic_dec(&cm_node->listener->pend_accepts_cnt); - cm_node->accept_pend = 0; - BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0); - } - - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, (jiffies + NES_SHORT_TIME)); - - return 0; -} - - -/** - * nes_addr_resolve_neigh - */ -static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpindex) -{ - struct rtable *rt; - struct neighbour *neigh; - int rc = arpindex; - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - __be32 dst_ipaddr = htonl(dst_ip); - - rt = ip_route_output(&init_net, dst_ipaddr, nesvnic->local_ipaddr, 0, 0); - if (IS_ERR(rt)) { - printk(KERN_ERR "%s: ip_route_output_key failed for 0x%08X\n", - __func__, dst_ip); - return rc; - } - - neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr); - - rcu_read_lock(); - if (neigh) { - if (neigh->nud_state & NUD_VALID) { - nes_debug(NES_DBG_CM, "Neighbor MAC address for 0x%08X" - " is %pM, Gateway is 0x%08X \n", dst_ip, - neigh->ha, ntohl(rt->rt_gw4)); - - if (arpindex >= 0) { - if (ether_addr_equal(nesadapter->arp_table[arpindex].mac_addr, neigh->ha)) { - /* Mac address same as in nes_arp_table */ - goto out; - } - - nes_manage_arp_cache(nesvnic->netdev, - nesadapter->arp_table[arpindex].mac_addr, - dst_ip, NES_ARP_DELETE); - } - - nes_manage_arp_cache(nesvnic->netdev, neigh->ha, - dst_ip, NES_ARP_ADD); - rc = nes_arp_table(nesvnic->nesdev, dst_ip, NULL, - NES_ARP_RESOLVE); - } else { - neigh_event_send(neigh, NULL); - } - } -out: - rcu_read_unlock(); - - if (neigh) - neigh_release(neigh); - - ip_rt_put(rt); - return rc; -} - -/** - * make_cm_node - create a new instance of a cm node - */ -static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct nes_cm_info *cm_info, - struct nes_cm_listener *listener) -{ - struct nes_cm_node *cm_node; - int oldarpindex = 0; - int arpindex = 0; - struct nes_device *nesdev; - struct nes_adapter *nesadapter; - - /* create an hte and cm_node for this instance */ - cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); - if (!cm_node) - return NULL; - - /* set our node specific transport info */ - if (listener) { - cm_node->loc_addr = listener->loc_addr; - cm_node->loc_port = listener->loc_port; - } else { - cm_node->loc_addr = cm_info->loc_addr; - cm_node->loc_port = cm_info->loc_port; - } - cm_node->rem_addr = cm_info->rem_addr; - cm_node->rem_port = cm_info->rem_port; - - cm_node->mpa_frame_rev = mpa_version; - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - cm_node->mpav2_ird_ord = 0; - cm_node->ird_size = 0; - cm_node->ord_size = 0; - - nes_debug(NES_DBG_CM, "Make node addresses : loc = %pI4:%x, rem = %pI4:%x\n", - &cm_node->loc_addr, cm_node->loc_port, - &cm_node->rem_addr, cm_node->rem_port); - cm_node->listener = listener; - if (listener) - cm_node->tos = listener->tos; - cm_node->netdev = nesvnic->netdev; - cm_node->cm_id = cm_info->cm_id; - memcpy(cm_node->loc_mac, nesvnic->netdev->dev_addr, ETH_ALEN); - - nes_debug(NES_DBG_CM, "listener=%p, cm_id=%p\n", cm_node->listener, - cm_node->cm_id); - - spin_lock_init(&cm_node->retrans_list_lock); - - cm_node->loopbackpartner = NULL; - atomic_set(&cm_node->ref_count, 1); - /* associate our parent CM core */ - cm_node->cm_core = cm_core; - cm_node->tcp_cntxt.loc_id = NES_CM_DEF_LOCAL_ID; - cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->tcp_cntxt.rcv_wnd = NES_CM_DEFAULT_RCV_WND_SCALED >> - NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr), - htonl(cm_node->rem_addr), - htons(cm_node->loc_port), - htons(cm_node->rem_port)); - cm_node->tcp_cntxt.mss = nesvnic->max_frame_size - sizeof(struct iphdr) - - sizeof(struct tcphdr) - ETH_HLEN - VLAN_HLEN; - cm_node->tcp_cntxt.rcv_nxt = 0; - /* get a unique session ID , add thread_id to an upcounter to handle race */ - atomic_inc(&cm_core->node_cnt); - cm_node->conn_type = cm_info->conn_type; - cm_node->apbvt_set = 0; - cm_node->accept_pend = 0; - - cm_node->nesvnic = nesvnic; - /* get some device handles, for arp lookup */ - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - - cm_node->loopbackpartner = NULL; - - /* get the mac addr for the remote node */ - oldarpindex = nes_arp_table(nesdev, cm_node->rem_addr, - NULL, NES_ARP_RESOLVE); - arpindex = nes_addr_resolve_neigh(nesvnic, cm_node->rem_addr, - oldarpindex); - if (arpindex < 0) { - kfree(cm_node); - return NULL; - } - - /* copy the mac addr to node context */ - memcpy(cm_node->rem_mac, nesadapter->arp_table[arpindex].mac_addr, ETH_ALEN); - nes_debug(NES_DBG_CM, "Remote mac addr from arp table: %pM\n", - cm_node->rem_mac); - - add_hte_node(cm_core, cm_node); - atomic_inc(&cm_nodes_created); - - return cm_node; -} - - -/** - * add_ref_cm_node - destroy an instance of a cm node - */ -static int add_ref_cm_node(struct nes_cm_node *cm_node) -{ - atomic_inc(&cm_node->ref_count); - return 0; -} - - -/** - * rem_ref_cm_node - destroy an instance of a cm node - */ -static int rem_ref_cm_node(struct nes_cm_core *cm_core, - struct nes_cm_node *cm_node) -{ - unsigned long flags; - struct nes_qp *nesqp; - - if (!cm_node) - return -EINVAL; - - spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags); - if (atomic_dec_return(&cm_node->ref_count)) { - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - return 0; - } - list_del(&cm_node->list); - atomic_dec(&cm_core->ht_node_cnt); - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - - /* if the node is destroyed before connection was accelerated */ - if (!cm_node->accelerated && cm_node->accept_pend) { - BUG_ON(!cm_node->listener); - atomic_dec(&cm_node->listener->pend_accepts_cnt); - BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0); - } - WARN_ON(cm_node->send_entry); - if (cm_node->recv_entry) - handle_recv_entry(cm_node, 0); - if (cm_node->listener) { - mini_cm_dec_refcnt_listen(cm_core, cm_node->listener, 0); - } else { - if (cm_node->apbvt_set && cm_node->nesvnic) { - nes_manage_apbvt(cm_node->nesvnic, cm_node->loc_port, - PCI_FUNC(cm_node->nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - } - nes_debug(NES_DBG_NLMSG, "Delete APBVT loc_port = %04X\n", - cm_node->loc_port); - } - - atomic_dec(&cm_core->node_cnt); - atomic_inc(&cm_nodes_destroyed); - nesqp = cm_node->nesqp; - if (nesqp) { - nesqp->cm_node = NULL; - nes_rem_ref(&nesqp->ibqp); - cm_node->nesqp = NULL; - } - - kfree(cm_node); - return 0; -} - -/** - * process_options - */ -static int process_options(struct nes_cm_node *cm_node, u8 *optionsloc, - u32 optionsize, u32 syn_packet) -{ - u32 tmp; - u32 offset = 0; - union all_known_options *all_options; - char got_mss_option = 0; - - while (offset < optionsize) { - all_options = (union all_known_options *)(optionsloc + offset); - switch (all_options->as_base.optionnum) { - case OPTION_NUMBER_END: - offset = optionsize; - break; - case OPTION_NUMBER_NONE: - offset += 1; - continue; - case OPTION_NUMBER_MSS: - nes_debug(NES_DBG_CM, "%s: MSS Length: %d Offset: %d " - "Size: %d\n", __func__, - all_options->as_mss.length, offset, optionsize); - got_mss_option = 1; - if (all_options->as_mss.length != 4) { - return 1; - } else { - tmp = ntohs(all_options->as_mss.mss); - if (tmp > 0 && tmp < - cm_node->tcp_cntxt.mss) - cm_node->tcp_cntxt.mss = tmp; - } - break; - case OPTION_NUMBER_WINDOW_SCALE: - cm_node->tcp_cntxt.snd_wscale = - all_options->as_windowscale.shiftcount; - break; - default: - nes_debug(NES_DBG_CM, "TCP Option not understood: %x\n", - all_options->as_base.optionnum); - break; - } - offset += all_options->as_base.length; - } - if ((!got_mss_option) && (syn_packet)) - cm_node->tcp_cntxt.mss = NES_CM_DEFAULT_MSS; - return 0; -} - -static void drop_packet(struct sk_buff *skb) -{ - atomic_inc(&cm_accel_dropped_pkts); - dev_kfree_skb_any(skb); -} - -static void handle_fin_pkt(struct nes_cm_node *cm_node) -{ - nes_debug(NES_DBG_CM, "Received FIN, cm_node = %p, state = %u. " - "refcnt=%d\n", cm_node, cm_node->state, - atomic_read(&cm_node->ref_count)); - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_MPAREJ_RCVD: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_LAST_ACK; - send_fin(cm_node, NULL); - break; - case NES_CM_STATE_MPAREQ_SENT: - create_event(cm_node, NES_CM_EVENT_ABORTED); - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - add_ref_cm_node(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_FIN_WAIT1: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSING; - send_ack(cm_node, NULL); - /* Wait for ACK as this is simultaneous close.. - * After we receive ACK, do not send anything.. - * Just rm the node.. Done.. */ - break; - case NES_CM_STATE_FIN_WAIT2: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_TIME_WAIT; - send_ack(cm_node, NULL); - schedule_nes_timer(cm_node, NULL, NES_TIMER_TYPE_CLOSE, 1, 0); - break; - case NES_CM_STATE_TIME_WAIT: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_node->cm_core, cm_node); - break; - case NES_CM_STATE_TSA: - default: - nes_debug(NES_DBG_CM, "Error Rcvd FIN for node-%p state = %d\n", - cm_node, cm_node->state); - break; - } -} - - -static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - - int reset = 0; /* whether to send reset in case of err.. */ - atomic_inc(&cm_resets_recvd); - nes_debug(NES_DBG_CM, "Received Reset, cm_node = %p, state = %u." - " refcnt=%d\n", cm_node, cm_node->state, - atomic_read(&cm_node->ref_count)); - cleanup_retrans_entry(cm_node); - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V2: - cm_node->mpa_frame_rev = IETF_MPA_V1; - /* send a syn and goto syn sent state */ - cm_node->state = NES_CM_STATE_SYN_SENT; - if (send_syn(cm_node, 0, NULL)) { - active_open_err(cm_node, skb, reset); - } - break; - case IETF_MPA_V1: - default: - active_open_err(cm_node, skb, reset); - break; - } - break; - case NES_CM_STATE_MPAREQ_RCVD: - atomic_inc(&cm_node->passive_state); - dev_kfree_skb_any(skb); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_LISTENING: - nes_debug(NES_DBG_CM, "Bad state %s[%u]\n", __func__, __LINE__); - passive_open_err(cm_node, skb, reset); - break; - case NES_CM_STATE_TSA: - active_open_err(cm_node, skb, reset); - break; - case NES_CM_STATE_CLOSED: - drop_packet(skb); - break; - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_LAST_ACK: - cm_node->cm_id->rem_ref(cm_node->cm_id); - /* fall through */ - case NES_CM_STATE_TIME_WAIT: - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_node->cm_core, cm_node); - drop_packet(skb); - break; - default: - drop_packet(skb); - break; - } -} - - -static void handle_rcv_mpa(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret = 0; - int datasize = skb->len; - u8 *dataloc = skb->data; - - enum nes_cm_event_type type = NES_CM_EVENT_UNKNOWN; - u32 res_type; - - ret = parse_mpa(cm_node, dataloc, &res_type, datasize); - if (ret) { - nes_debug(NES_DBG_CM, "didn't like MPA Request\n"); - if (cm_node->state == NES_CM_STATE_MPAREQ_SENT) { - nes_debug(NES_DBG_CM, "%s[%u] create abort for " - "cm_node=%p listener=%p state=%d\n", __func__, - __LINE__, cm_node, cm_node->listener, - cm_node->state); - active_open_err(cm_node, skb, 1); - } else { - passive_open_err(cm_node, skb, 1); - } - return; - } - - switch (cm_node->state) { - case NES_CM_STATE_ESTABLISHED: - if (res_type == NES_MPA_REQUEST_REJECT) - /*BIG problem as we are receiving the MPA.. So should - * not be REJECT.. This is Passive Open.. We can - * only receive it Reject for Active Open...*/ - WARN_ON(1); - cm_node->state = NES_CM_STATE_MPAREQ_RCVD; - type = NES_CM_EVENT_MPA_REQ; - atomic_set(&cm_node->passive_state, - NES_PASSIVE_STATE_INDICATED); - break; - case NES_CM_STATE_MPAREQ_SENT: - cleanup_retrans_entry(cm_node); - if (res_type == NES_MPA_REQUEST_REJECT) { - type = NES_CM_EVENT_MPA_REJECT; - cm_node->state = NES_CM_STATE_MPAREJ_RCVD; - } else { - type = NES_CM_EVENT_CONNECTED; - cm_node->state = NES_CM_STATE_TSA; - } - send_ack(cm_node, NULL); - break; - default: - WARN_ON(1); - break; - } - dev_kfree_skb_any(skb); - create_event(cm_node, type); -} - -static void indicate_pkt_err(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - active_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_SYN_RCVD: - passive_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_TSA: - default: - drop_packet(skb); - } -} - -static int check_syn(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb) -{ - int err; - - err = ((ntohl(tcph->ack_seq) == cm_node->tcp_cntxt.loc_seq_num)) ? 0 : 1; - if (err) - active_open_err(cm_node, skb, 1); - - return err; -} - -static int check_seq(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb) -{ - int err = 0; - u32 seq; - u32 ack_seq; - u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; - u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; - u32 rcv_wnd; - - seq = ntohl(tcph->seq); - ack_seq = ntohl(tcph->ack_seq); - rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; - if (ack_seq != loc_seq_num) - err = 1; - else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) - err = 1; - if (err) { - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - indicate_pkt_err(cm_node, skb); - nes_debug(NES_DBG_CM, "seq ERROR cm_node =%p seq=0x%08X " - "rcv_nxt=0x%08X rcv_wnd=0x%x\n", cm_node, seq, rcv_nxt, - rcv_wnd); - } - return err; -} - -/* - * handle_syn_pkt() is for Passive node. The syn packet is received when a node - * is created with a listener or it may comein as rexmitted packet which in - * that case will be just dropped. - */ -static void handle_syn_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int ret; - u32 inc_sequence; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - skb_trim(skb, 0); - inc_sequence = ntohl(tcph->seq); - - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - /* Rcvd syn on active open connection*/ - active_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_LISTENING: - /* Passive OPEN */ - if (atomic_read(&cm_node->listener->pend_accepts_cnt) > - cm_node->listener->backlog) { - nes_debug(NES_DBG_CM, "drop syn due to backlog " - "pressure \n"); - cm_backlog_drops++; - passive_open_err(cm_node, skb, 0); - break; - } - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, - 1); - if (ret) { - passive_open_err(cm_node, skb, 0); - /* drop pkt */ - break; - } - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - BUG_ON(cm_node->send_entry); - cm_node->accept_pend = 1; - atomic_inc(&cm_node->listener->pend_accepts_cnt); - - cm_node->state = NES_CM_STATE_SYN_RCVD; - send_syn(cm_node, 1, skb); - break; - case NES_CM_STATE_CLOSED: - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_TSA: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_MPAREQ_RCVD: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_CLOSING: - case NES_CM_STATE_UNKNOWN: - default: - drop_packet(skb); - break; - } -} - -static void handle_synack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int ret; - u32 inc_sequence; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - skb_trim(skb, 0); - inc_sequence = ntohl(tcph->seq); - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - cleanup_retrans_entry(cm_node); - /* active open */ - if (check_syn(cm_node, tcph, skb)) - return; - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - /* setup options */ - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 0); - if (ret) { - nes_debug(NES_DBG_CM, "cm_node=%p tcp_options failed\n", - cm_node); - break; - } - cleanup_retrans_entry(cm_node); - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - send_mpa_request(cm_node, skb); - cm_node->state = NES_CM_STATE_MPAREQ_SENT; - break; - case NES_CM_STATE_MPAREQ_RCVD: - /* passive open, so should not be here */ - passive_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_LISTENING: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - send_reset(cm_node, skb); - break; - case NES_CM_STATE_CLOSED: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_TSA: - case NES_CM_STATE_CLOSING: - case NES_CM_STATE_UNKNOWN: - case NES_CM_STATE_MPAREQ_SENT: - default: - drop_packet(skb); - break; - } -} - -static int handle_ack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int datasize = 0; - u32 inc_sequence; - int ret = 0; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - - if (check_seq(cm_node, tcph, skb)) - return -EINVAL; - - skb_pull(skb, tcph->doff << 2); - inc_sequence = ntohl(tcph->seq); - datasize = skb->len; - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - /* Passive OPEN */ - cleanup_retrans_entry(cm_node); - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 1); - if (ret) - break; - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - cm_node->state = NES_CM_STATE_ESTABLISHED; - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { /* rcvd ACK only */ - dev_kfree_skb_any(skb); - } - break; - case NES_CM_STATE_ESTABLISHED: - /* Passive OPEN */ - cleanup_retrans_entry(cm_node); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { - drop_packet(skb); - } - break; - case NES_CM_STATE_MPAREQ_SENT: - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { /* Could be just an ack pkt.. */ - dev_kfree_skb_any(skb); - } - break; - case NES_CM_STATE_LISTENING: - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - send_reset(cm_node, skb); - break; - case NES_CM_STATE_CLOSED: - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_CLOSING: - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - cm_node->cm_id->rem_ref(cm_node->cm_id); - rem_ref_cm_node(cm_node->cm_core, cm_node); - drop_packet(skb); - break; - case NES_CM_STATE_FIN_WAIT1: - cleanup_retrans_entry(cm_node); - drop_packet(skb); - cm_node->state = NES_CM_STATE_FIN_WAIT2; - break; - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_TSA: - case NES_CM_STATE_MPAREQ_RCVD: - case NES_CM_STATE_UNKNOWN: - default: - cleanup_retrans_entry(cm_node); - drop_packet(skb); - break; - } - return ret; -} - - - -static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb, int optionsize, int passive) -{ - u8 *optionsloc = (u8 *)&tcph[1]; - - if (optionsize) { - if (process_options(cm_node, optionsloc, optionsize, - (u32)tcph->syn)) { - nes_debug(NES_DBG_CM, "%s: Node %p, Sending RESET\n", - __func__, cm_node); - if (passive) - passive_open_err(cm_node, skb, 1); - else - active_open_err(cm_node, skb, 1); - return 1; - } - } - - cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) << - cm_node->tcp_cntxt.snd_wscale; - - if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) - cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; - return 0; -} - -/* - * active_open_err() will send reset() if flag set.. - * It will also send ABORT event. - */ -static void active_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb, - int reset) -{ - cleanup_retrans_entry(cm_node); - if (reset) { - nes_debug(NES_DBG_CM, "ERROR active err called for cm_node=%p, " - "state=%d\n", cm_node, cm_node->state); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - } else { - dev_kfree_skb_any(skb); - } - - cm_node->state = NES_CM_STATE_CLOSED; - create_event(cm_node, NES_CM_EVENT_ABORTED); -} - -/* - * passive_open_err() will either do a reset() or will free up the skb and - * remove the cm_node. - */ -static void passive_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb, - int reset) -{ - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - if (reset) { - nes_debug(NES_DBG_CM, "passive_open_err sending RST for " - "cm_node=%p state =%d\n", cm_node, cm_node->state); - send_reset(cm_node, skb); - } else { - dev_kfree_skb_any(skb); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } -} - -/* - * free_retrans_entry() routines assumes that the retrans_list_lock has - * been acquired before calling. - */ -static void free_retrans_entry(struct nes_cm_node *cm_node) -{ - struct nes_timer_entry *send_entry; - - send_entry = cm_node->send_entry; - if (send_entry) { - cm_node->send_entry = NULL; - dev_kfree_skb_any(send_entry->skb); - kfree(send_entry); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } -} - -static void cleanup_retrans_entry(struct nes_cm_node *cm_node) -{ - unsigned long flags; - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - free_retrans_entry(cm_node); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); -} - -/** - * process_packet - * Returns skb if to be freed, else it will return NULL if already used.. - */ -static void process_packet(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct nes_cm_core *cm_core) -{ - enum nes_tcpip_pkt_type pkt_type = NES_PKT_TYPE_UNKNOWN; - struct tcphdr *tcph = tcp_hdr(skb); - u32 fin_set = 0; - int ret = 0; - - skb_pull(skb, ip_hdr(skb)->ihl << 2); - - nes_debug(NES_DBG_CM, "process_packet: cm_node=%p state =%d syn=%d " - "ack=%d rst=%d fin=%d\n", cm_node, cm_node->state, tcph->syn, - tcph->ack, tcph->rst, tcph->fin); - - if (tcph->rst) { - pkt_type = NES_PKT_TYPE_RST; - } else if (tcph->syn) { - pkt_type = NES_PKT_TYPE_SYN; - if (tcph->ack) - pkt_type = NES_PKT_TYPE_SYNACK; - } else if (tcph->ack) { - pkt_type = NES_PKT_TYPE_ACK; - } - if (tcph->fin) - fin_set = 1; - - switch (pkt_type) { - case NES_PKT_TYPE_SYN: - handle_syn_pkt(cm_node, skb, tcph); - break; - case NES_PKT_TYPE_SYNACK: - handle_synack_pkt(cm_node, skb, tcph); - break; - case NES_PKT_TYPE_ACK: - ret = handle_ack_pkt(cm_node, skb, tcph); - if (fin_set && !ret) - handle_fin_pkt(cm_node); - break; - case NES_PKT_TYPE_RST: - handle_rst_pkt(cm_node, skb, tcph); - break; - default: - if ((fin_set) && (!check_seq(cm_node, tcph, skb))) - handle_fin_pkt(cm_node); - drop_packet(skb); - break; - } -} - -/** - * mini_cm_listen - create a listen node with params - */ -static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct nes_cm_info *cm_info) -{ - struct nes_cm_listener *listener; - unsigned long flags; - - nes_debug(NES_DBG_CM, "Search for 0x%08x : 0x%04x\n", - cm_info->loc_addr, cm_info->loc_port); - - /* cannot have multiple matching listeners */ - listener = find_listener(cm_core, cm_info->loc_addr, cm_info->loc_port, - NES_CM_LISTENER_EITHER_STATE); - - if (listener && listener->listener_state == NES_CM_LISTENER_ACTIVE_STATE) { - /* find automatically incs ref count ??? */ - atomic_dec(&listener->ref_count); - nes_debug(NES_DBG_CM, "Not creating listener since it already exists\n"); - return NULL; - } - - if (!listener) { - /* create a CM listen node (1/2 node to compare incoming traffic to) */ - listener = kzalloc(sizeof(*listener), GFP_ATOMIC); - if (!listener) - return NULL; - - listener->loc_addr = cm_info->loc_addr; - listener->loc_port = cm_info->loc_port; - listener->reused_node = 0; - - atomic_set(&listener->ref_count, 1); - } - /* pasive case */ - /* find already inc'ed the ref count */ - else { - listener->reused_node = 1; - } - - listener->cm_id = cm_info->cm_id; - atomic_set(&listener->pend_accepts_cnt, 0); - listener->cm_core = cm_core; - listener->nesvnic = nesvnic; - atomic_inc(&cm_core->node_cnt); - - listener->conn_type = cm_info->conn_type; - listener->backlog = cm_info->backlog; - listener->listener_state = NES_CM_LISTENER_ACTIVE_STATE; - - if (!listener->reused_node) { - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_add(&listener->list, &cm_core->listen_list.list); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - atomic_inc(&cm_core->listen_node_cnt); - } - - nes_debug(NES_DBG_CM, "Api - listen(): addr=0x%08X, port=0x%04x," - " listener = %p, backlog = %d, cm_id = %p.\n", - cm_info->loc_addr, cm_info->loc_port, - listener, listener->backlog, listener->cm_id); - - return listener; -} - - -/** - * mini_cm_connect - make a connection node with params - */ -static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, u16 private_data_len, - void *private_data, struct nes_cm_info *cm_info) -{ - int ret = 0; - struct nes_cm_node *cm_node; - struct nes_cm_listener *loopbackremotelistener; - struct nes_cm_node *loopbackremotenode; - struct nes_cm_info loopback_cm_info; - u8 *start_buff; - - /* create a CM connection node */ - cm_node = make_cm_node(cm_core, nesvnic, cm_info, NULL); - if (!cm_node) - return NULL; - - /* set our node side to client (active) side */ - cm_node->tcp_cntxt.client = 1; - cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE; - - if (cm_info->loc_addr == cm_info->rem_addr) { - loopbackremotelistener = find_listener(cm_core, - cm_node->loc_addr, cm_node->rem_port, - NES_CM_LISTENER_ACTIVE_STATE); - if (loopbackremotelistener == NULL) { - create_event(cm_node, NES_CM_EVENT_ABORTED); - } else { - loopback_cm_info = *cm_info; - loopback_cm_info.loc_port = cm_info->rem_port; - loopback_cm_info.rem_port = cm_info->loc_port; - loopback_cm_info.loc_port = - cm_info->rem_port; - loopback_cm_info.rem_port = - cm_info->loc_port; - loopback_cm_info.cm_id = loopbackremotelistener->cm_id; - loopbackremotenode = make_cm_node(cm_core, nesvnic, - &loopback_cm_info, loopbackremotelistener); - if (!loopbackremotenode) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - return NULL; - } - atomic_inc(&cm_loopbacks); - loopbackremotenode->loopbackpartner = cm_node; - loopbackremotenode->tcp_cntxt.rcv_wscale = - NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->loopbackpartner = loopbackremotenode; - memcpy(loopbackremotenode->mpa_frame_buf, private_data, - private_data_len); - loopbackremotenode->mpa_frame_size = private_data_len; - - /* we are done handling this state. */ - /* set node to a TSA state */ - cm_node->state = NES_CM_STATE_TSA; - cm_node->tcp_cntxt.rcv_nxt = - loopbackremotenode->tcp_cntxt.loc_seq_num; - loopbackremotenode->tcp_cntxt.rcv_nxt = - cm_node->tcp_cntxt.loc_seq_num; - cm_node->tcp_cntxt.max_snd_wnd = - loopbackremotenode->tcp_cntxt.rcv_wnd; - loopbackremotenode->tcp_cntxt.max_snd_wnd = - cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wnd = - loopbackremotenode->tcp_cntxt.rcv_wnd; - loopbackremotenode->tcp_cntxt.snd_wnd = - cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wscale = - loopbackremotenode->tcp_cntxt.rcv_wscale; - loopbackremotenode->tcp_cntxt.snd_wscale = - cm_node->tcp_cntxt.rcv_wscale; - loopbackremotenode->state = NES_CM_STATE_MPAREQ_RCVD; - create_event(loopbackremotenode, NES_CM_EVENT_MPA_REQ); - } - return cm_node; - } - - start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2); - cm_node->mpa_frame_size = private_data_len; - - memcpy(start_buff, private_data, private_data_len); - - /* send a syn and goto syn sent state */ - cm_node->state = NES_CM_STATE_SYN_SENT; - ret = send_syn(cm_node, 0, NULL); - - if (ret) { - /* error in sending the syn free up the cm_node struct */ - nes_debug(NES_DBG_CM, "Api - connect() FAILED: dest " - "addr=0x%08X, port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_addr, cm_node->rem_port, cm_node, - cm_node->cm_id); - rem_ref_cm_node(cm_node->cm_core, cm_node); - cm_node = NULL; - } - - if (cm_node) { - nes_debug(NES_DBG_CM, "Api - connect(): dest addr=0x%08X," - "port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_addr, cm_node->rem_port, cm_node, - cm_node->cm_id); - } - - return cm_node; -} - - -/** - * mini_cm_accept - accept a connection - * This function is never called - */ -static int mini_cm_accept(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - return 0; -} - - -/** - * mini_cm_reject - reject and teardown a connection - */ -static int mini_cm_reject(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - int ret = 0; - int err = 0; - int passive_state; - struct nes_cm_event event; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct nes_cm_node *loopback = cm_node->loopbackpartner; - - nes_debug(NES_DBG_CM, "%s cm_node=%p type=%d state=%d\n", - __func__, cm_node, cm_node->tcp_cntxt.client, cm_node->state); - - if (cm_node->tcp_cntxt.client) - return ret; - cleanup_retrans_entry(cm_node); - - if (!loopback) { - passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) { - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_core, cm_node); - } else { - if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) { - rem_ref_cm_node(cm_core, cm_node); - } else { - ret = send_mpa_reject(cm_node); - if (ret) { - cm_node->state = NES_CM_STATE_CLOSED; - err = send_reset(cm_node, NULL); - if (err) - WARN_ON(1); - } else { - cm_id->add_ref(cm_id); - } - } - } - } else { - cm_node->cm_id = NULL; - if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) { - rem_ref_cm_node(cm_core, cm_node); - rem_ref_cm_node(cm_core, loopback); - } else { - event.cm_node = loopback; - event.cm_info.rem_addr = loopback->rem_addr; - event.cm_info.loc_addr = loopback->loc_addr; - event.cm_info.rem_port = loopback->rem_port; - event.cm_info.loc_port = loopback->loc_port; - event.cm_info.cm_id = loopback->cm_id; - cm_event_mpa_reject(&event); - rem_ref_cm_node(cm_core, cm_node); - loopback->state = NES_CM_STATE_CLOSING; - - cm_id = loopback->cm_id; - rem_ref_cm_node(cm_core, loopback); - cm_id->rem_ref(cm_id); - } - } - - return ret; -} - - -/** - * mini_cm_close - */ -static int mini_cm_close(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - int ret = 0; - - if (!cm_core || !cm_node) - return -EINVAL; - - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_ONE_SIDE_ESTABLISHED: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_ACCEPTING: - case NES_CM_STATE_MPAREQ_SENT: - case NES_CM_STATE_MPAREQ_RCVD: - cleanup_retrans_entry(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_CLOSE_WAIT: - cm_node->state = NES_CM_STATE_LAST_ACK; - send_fin(cm_node, NULL); - break; - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_TIME_WAIT: - case NES_CM_STATE_CLOSING: - ret = -1; - break; - case NES_CM_STATE_LISTENING: - cleanup_retrans_entry(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_MPAREJ_RCVD: - case NES_CM_STATE_UNKNOWN: - case NES_CM_STATE_INITED: - case NES_CM_STATE_CLOSED: - case NES_CM_STATE_LISTENER_DESTROYED: - ret = rem_ref_cm_node(cm_core, cm_node); - break; - case NES_CM_STATE_TSA: - if (cm_node->send_entry) - printk(KERN_ERR "ERROR Close got called from STATE_TSA " - "send_entry=%p\n", cm_node->send_entry); - ret = rem_ref_cm_node(cm_core, cm_node); - break; - } - return ret; -} - - -/** - * recv_pkt - recv an ETHERNET packet, and process it through CM - * node state machine - */ -static int mini_cm_recv_pkt(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct sk_buff *skb) -{ - struct nes_cm_node *cm_node = NULL; - struct nes_cm_listener *listener = NULL; - struct iphdr *iph; - struct tcphdr *tcph; - struct nes_cm_info nfo; - int skb_handled = 1; - __be32 tmp_daddr, tmp_saddr; - - if (!skb) - return 0; - if (skb->len < sizeof(struct iphdr) + sizeof(struct tcphdr)) - return 0; - - iph = (struct iphdr *)skb->data; - tcph = (struct tcphdr *)(skb->data + sizeof(struct iphdr)); - - nfo.loc_addr = ntohl(iph->daddr); - nfo.loc_port = ntohs(tcph->dest); - nfo.rem_addr = ntohl(iph->saddr); - nfo.rem_port = ntohs(tcph->source); - - tmp_daddr = cpu_to_be32(iph->daddr); - tmp_saddr = cpu_to_be32(iph->saddr); - - nes_debug(NES_DBG_CM, "Received packet: dest=%pI4:0x%04X src=%pI4:0x%04X\n", - &tmp_daddr, tcph->dest, &tmp_saddr, tcph->source); - - do { - cm_node = find_node(cm_core, - nfo.rem_port, nfo.rem_addr, - nfo.loc_port, nfo.loc_addr); - - if (!cm_node) { - /* Only type of packet accepted are for */ - /* the PASSIVE open (syn only) */ - if ((!tcph->syn) || (tcph->ack)) { - skb_handled = 0; - break; - } - listener = find_listener(cm_core, nfo.loc_addr, - nfo.loc_port, - NES_CM_LISTENER_ACTIVE_STATE); - if (!listener) { - nfo.cm_id = NULL; - nfo.conn_type = 0; - nes_debug(NES_DBG_CM, "Unable to find listener for the pkt\n"); - skb_handled = 0; - break; - } - nfo.cm_id = listener->cm_id; - nfo.conn_type = listener->conn_type; - cm_node = make_cm_node(cm_core, nesvnic, &nfo, - listener); - if (!cm_node) { - nes_debug(NES_DBG_CM, "Unable to allocate " - "node\n"); - cm_packets_dropped++; - atomic_dec(&listener->ref_count); - dev_kfree_skb_any(skb); - break; - } - if (!tcph->rst && !tcph->fin) { - cm_node->state = NES_CM_STATE_LISTENING; - } else { - cm_packets_dropped++; - rem_ref_cm_node(cm_core, cm_node); - dev_kfree_skb_any(skb); - break; - } - add_ref_cm_node(cm_node); - } else if (cm_node->state == NES_CM_STATE_TSA) { - if (cm_node->nesqp->pau_mode) - nes_queue_mgt_skbs(skb, nesvnic, cm_node->nesqp); - else { - rem_ref_cm_node(cm_core, cm_node); - atomic_inc(&cm_accel_dropped_pkts); - dev_kfree_skb_any(skb); - } - break; - } - skb_reset_network_header(skb); - skb_set_transport_header(skb, sizeof(*tcph)); - skb->len = ntohs(iph->tot_len); - process_packet(cm_node, skb, cm_core); - rem_ref_cm_node(cm_core, cm_node); - } while (0); - return skb_handled; -} - - -/** - * nes_cm_alloc_core - allocate a top level instance of a cm core - */ -static struct nes_cm_core *nes_cm_alloc_core(void) -{ - struct nes_cm_core *cm_core; - - /* setup the CM core */ - /* alloc top level core control structure */ - cm_core = kzalloc(sizeof(*cm_core), GFP_KERNEL); - if (!cm_core) - return NULL; - - INIT_LIST_HEAD(&cm_core->connected_nodes); - timer_setup(&cm_core->tcp_timer, nes_cm_timer_tick, 0); - - cm_core->mtu = NES_CM_DEFAULT_MTU; - cm_core->state = NES_CM_STATE_INITED; - cm_core->free_tx_pkt_max = NES_CM_DEFAULT_FREE_PKTS; - - atomic_set(&cm_core->events_posted, 0); - - cm_core->api = &nes_cm_api; - - spin_lock_init(&cm_core->ht_lock); - spin_lock_init(&cm_core->listen_list_lock); - - INIT_LIST_HEAD(&cm_core->listen_list.list); - - nes_debug(NES_DBG_CM, "Init CM Core completed -- cm_core=%p\n", cm_core); - - nes_debug(NES_DBG_CM, "Enable QUEUE EVENTS\n"); - cm_core->event_wq = alloc_ordered_workqueue("nesewq", 0); - if (!cm_core->event_wq) - goto out_free_cmcore; - cm_core->post_event = nes_cm_post_event; - nes_debug(NES_DBG_CM, "Enable QUEUE DISCONNECTS\n"); - cm_core->disconn_wq = alloc_ordered_workqueue("nesdwq", 0); - if (!cm_core->disconn_wq) - goto out_free_wq; - - print_core(cm_core); - return cm_core; - -out_free_wq: - destroy_workqueue(cm_core->event_wq); -out_free_cmcore: - kfree(cm_core); - return NULL; -} - - -/** - * mini_cm_dealloc_core - deallocate a top level instance of a cm core - */ -static int mini_cm_dealloc_core(struct nes_cm_core *cm_core) -{ - nes_debug(NES_DBG_CM, "De-Alloc CM Core (%p)\n", cm_core); - - if (!cm_core) - return -EINVAL; - - barrier(); - - if (timer_pending(&cm_core->tcp_timer)) - del_timer(&cm_core->tcp_timer); - - destroy_workqueue(cm_core->event_wq); - destroy_workqueue(cm_core->disconn_wq); - nes_debug(NES_DBG_CM, "\n"); - kfree(cm_core); - - return 0; -} - - -/** - * mini_cm_get - */ -static int mini_cm_get(struct nes_cm_core *cm_core) -{ - return cm_core->state; -} - - -/** - * mini_cm_set - */ -static int mini_cm_set(struct nes_cm_core *cm_core, u32 type, u32 value) -{ - int ret = 0; - - switch (type) { - case NES_CM_SET_PKT_SIZE: - cm_core->mtu = value; - break; - case NES_CM_SET_FREE_PKT_Q_SIZE: - cm_core->free_tx_pkt_max = value; - break; - default: - /* unknown set option */ - ret = -EINVAL; - } - - return ret; -} - - -/** - * nes_cm_init_tsa_conn setup HW; MPA frames must be - * successfully exchanged when this is called - */ -static int nes_cm_init_tsa_conn(struct nes_qp *nesqp, struct nes_cm_node *cm_node) -{ - int ret = 0; - - if (!nesqp) - return -EINVAL; - - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_IPV4 | - NES_QPCONTEXT_MISC_NO_NAGLE | NES_QPCONTEXT_MISC_DO_NOT_FRAG | - NES_QPCONTEXT_MISC_DROS); - - if (cm_node->tcp_cntxt.snd_wscale || cm_node->tcp_cntxt.rcv_wscale) - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WSCALE); - - nesqp->nesqp_context->misc2 |= cpu_to_le32(64 << NES_QPCONTEXT_MISC2_TTL_SHIFT); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - cm_node->tos << NES_QPCONTEXT_MISC2_TOS_SHIFT); - - nesqp->nesqp_context->mss |= cpu_to_le32(((u32)cm_node->tcp_cntxt.mss) << 16); - - nesqp->nesqp_context->tcp_state_flow_label |= cpu_to_le32( - (u32)NES_QPCONTEXT_TCPSTATE_EST << NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32( - (cm_node->tcp_cntxt.snd_wscale << NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT) & - NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32( - (cm_node->tcp_cntxt.rcv_wscale << NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT) & - NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK); - - nesqp->nesqp_context->keepalive = cpu_to_le32(0x80); - nesqp->nesqp_context->ts_recent = 0; - nesqp->nesqp_context->ts_age = 0; - nesqp->nesqp_context->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd); - nesqp->nesqp_context->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - nesqp->nesqp_context->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd << - cm_node->tcp_cntxt.rcv_wscale); - nesqp->nesqp_context->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->srtt = 0; - nesqp->nesqp_context->rttvar = cpu_to_le32(0x6); - nesqp->nesqp_context->ssthresh = cpu_to_le32(0x3FFFC000); - nesqp->nesqp_context->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss); - nesqp->nesqp_context->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - nesqp->nesqp_context->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->max_snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd); - - nes_debug(NES_DBG_CM, "QP%u: rcv_nxt = 0x%08X, snd_nxt = 0x%08X," - " Setting MSS to %u, PDWscale = 0x%08X, rcv_wnd = %u, context misc = 0x%08X.\n", - nesqp->hwqp.qp_id, le32_to_cpu(nesqp->nesqp_context->rcv_nxt), - le32_to_cpu(nesqp->nesqp_context->snd_nxt), - cm_node->tcp_cntxt.mss, le32_to_cpu(nesqp->nesqp_context->pd_index_wscale), - le32_to_cpu(nesqp->nesqp_context->rcv_wnd), - le32_to_cpu(nesqp->nesqp_context->misc)); - nes_debug(NES_DBG_CM, " snd_wnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->snd_wnd)); - nes_debug(NES_DBG_CM, " snd_cwnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->cwnd)); - nes_debug(NES_DBG_CM, " max_swnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->max_snd_wnd)); - - nes_debug(NES_DBG_CM, "Change cm_node state to TSA\n"); - cm_node->state = NES_CM_STATE_TSA; - - return ret; -} - - -/** - * nes_cm_disconn - */ -int nes_cm_disconn(struct nes_qp *nesqp) -{ - struct disconn_work *work; - - work = kzalloc(sizeof *work, GFP_ATOMIC); - if (!work) - return -ENOMEM; /* Timer will clean up */ - - nes_add_ref(&nesqp->ibqp); - work->nesqp = nesqp; - INIT_WORK(&work->work, nes_disconnect_worker); - queue_work(g_cm_core->disconn_wq, &work->work); - return 0; -} - - -/** - * nes_disconnect_worker - */ -static void nes_disconnect_worker(struct work_struct *work) -{ - struct disconn_work *dwork = container_of(work, struct disconn_work, work); - struct nes_qp *nesqp = dwork->nesqp; - - kfree(dwork); - nes_debug(NES_DBG_CM, "processing AEQE id 0x%04X for QP%u.\n", - nesqp->last_aeq, nesqp->hwqp.qp_id); - nes_cm_disconn_true(nesqp); - nes_rem_ref(&nesqp->ibqp); -} - - -/** - * nes_cm_disconn_true - */ -static int nes_cm_disconn_true(struct nes_qp *nesqp) -{ - unsigned long flags; - int ret = 0; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_vnic *nesvnic; - u16 last_ae; - u8 original_hw_tcp_state; - u8 original_ibqp_state; - int disconn_status = 0; - int issue_disconn = 0; - int issue_close = 0; - int issue_flush = 0; - u32 flush_q = NES_CQP_FLUSH_RQ; - struct ib_event ibevent; - - if (!nesqp) { - nes_debug(NES_DBG_CM, "disconnect_worker nesqp is NULL\n"); - return -1; - } - - spin_lock_irqsave(&nesqp->lock, flags); - cm_id = nesqp->cm_id; - /* make sure we havent already closed this connection */ - if (!cm_id) { - nes_debug(NES_DBG_CM, "QP%u disconnect_worker cmid is NULL\n", - nesqp->hwqp.qp_id); - spin_unlock_irqrestore(&nesqp->lock, flags); - return -1; - } - - nesvnic = to_nesvnic(nesqp->ibqp.device); - nes_debug(NES_DBG_CM, "Disconnecting QP%u\n", nesqp->hwqp.qp_id); - - original_hw_tcp_state = nesqp->hw_tcp_state; - original_ibqp_state = nesqp->ibqp_state; - last_ae = nesqp->last_aeq; - - if (nesqp->term_flags) { - issue_disconn = 1; - issue_close = 1; - nesqp->cm_id = NULL; - del_timer(&nesqp->terminate_timer); - if (nesqp->flush_issued == 0) { - nesqp->flush_issued = 1; - issue_flush = 1; - } - } else if ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) || - ((original_ibqp_state == IB_QPS_RTS) && - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { - issue_disconn = 1; - if (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET) - disconn_status = -ECONNRESET; - } - - if (((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSED) || - (original_hw_tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT) || - (last_ae == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) || - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { - issue_close = 1; - nesqp->cm_id = NULL; - if (nesqp->flush_issued == 0) { - nesqp->flush_issued = 1; - issue_flush = 1; - } - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - - if ((issue_flush) && (nesqp->destroyed == 0)) { - /* Flush the queue(s) */ - if (nesqp->hw_iwarp_state >= NES_AEQE_IWARP_STATE_TERMINATE) - flush_q |= NES_CQP_FLUSH_SQ; - flush_wqes(nesvnic->nesdev, nesqp, flush_q, 1); - - if (nesqp->term_flags) { - ibevent.device = nesqp->ibqp.device; - ibevent.event = nesqp->terminate_eventtype; - ibevent.element.qp = &nesqp->ibqp; - if (nesqp->ibqp.event_handler) - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - } - - if ((cm_id) && (cm_id->event_handler)) { - if (issue_disconn) { - atomic_inc(&cm_disconnects); - cm_event.event = IW_CM_EVENT_DISCONNECT; - cm_event.status = disconn_status; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - nes_debug(NES_DBG_CM, "Generating a CM Disconnect Event" - " for QP%u, SQ Head = %u, SQ Tail = %u. " - "cm_id = %p, refcount = %u.\n", - nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, - nesqp->hwqp.sq_tail, cm_id, - atomic_read(&nesqp->refcount)); - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_CM, "OFA CM event_handler " - "returned, ret=%d\n", ret); - } - - if (issue_close) { - atomic_inc(&cm_closes); - nes_disconnect(nesqp, 1); - - cm_id->provider_data = nesqp; - /* Send up the close complete event */ - cm_event.event = IW_CM_EVENT_CLOSE; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - cm_id->rem_ref(cm_id); - } - } - - return 0; -} - - -/** - * nes_disconnect - */ -static int nes_disconnect(struct nes_qp *nesqp, int abrupt) -{ - int ret = 0; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_ib_device *nesibdev; - - nesvnic = to_nesvnic(nesqp->ibqp.device); - if (!nesvnic) - return -EINVAL; - - nesdev = nesvnic->nesdev; - nesibdev = nesvnic->nesibdev; - - nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n", - netdev_refcnt_read(nesvnic->netdev)); - - if (nesqp->active_conn) { - - /* indicate this connection is NOT active */ - nesqp->active_conn = 0; - } else { - /* Need to free the Last Streaming Mode Message */ - if (nesqp->ietf_frame) { - if (nesqp->lsmm_mr) - nesibdev->ibdev.ops.dereg_mr(nesqp->lsmm_mr, - NULL); - pci_free_consistent(nesdev->pcidev, - nesqp->private_data_len + nesqp->ietf_frame_size, - nesqp->ietf_frame, nesqp->ietf_frame_pbase); - } - } - - /* close the CM node down if it is still active */ - if (nesqp->cm_node) { - nes_debug(NES_DBG_CM, "Call close API\n"); - - g_cm_core->api->close(g_cm_core, nesqp->cm_node); - } - - return ret; -} - - -/** - * nes_accept - */ -int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - u64 u64temp; - struct ib_qp *ibqp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_adapter *adapter; - struct ib_qp_attr attr; - struct iw_cm_event cm_event; - struct nes_hw_qp_wqe *wqe; - struct nes_v4_quad nes_quad; - u32 crc_value; - int ret; - int passive_state; - struct ib_mr *ibmr = NULL; - struct nes_pd *nespd; - u64 tagged_offset; - u8 mpa_frame_offset = 0; - struct ietf_mpa_v2 *mpa_v2_frame; - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - - ibqp = nes_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - - /* get all our handles */ - nesqp = to_nesqp(ibqp); - nesvnic = to_nesvnic(nesqp->ibqp.device); - nesdev = nesvnic->nesdev; - adapter = nesdev->nesadapter; - - cm_node = (struct nes_cm_node *)cm_id->provider_data; - nes_debug(NES_DBG_CM, "nes_accept: cm_node= %p nesvnic=%p, netdev=%p," - "%s\n", cm_node, nesvnic, nesvnic->netdev, - nesvnic->netdev->name); - - if (NES_CM_STATE_LISTENER_DESTROYED == cm_node->state) { - if (cm_node->loopbackpartner) - rem_ref_cm_node(cm_node->cm_core, cm_node->loopbackpartner); - rem_ref_cm_node(cm_node->cm_core, cm_node); - return -EINVAL; - } - - passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - return -ECONNRESET; - } - /* associate the node with the QP */ - nesqp->cm_node = (void *)cm_node; - cm_node->nesqp = nesqp; - - - nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n", - nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener); - atomic_inc(&cm_accepts); - - nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n", - netdev_refcnt_read(nesvnic->netdev)); - - nesqp->ietf_frame_size = sizeof(struct ietf_mpa_v2); - /* allocate the ietf frame and space for private data */ - nesqp->ietf_frame = pci_alloc_consistent(nesdev->pcidev, - nesqp->ietf_frame_size + conn_param->private_data_len, - &nesqp->ietf_frame_pbase); - - if (!nesqp->ietf_frame) { - nes_debug(NES_DBG_CM, "Unable to allocate memory for private data\n"); - return -ENOMEM; - } - mpa_v2_frame = (struct ietf_mpa_v2 *)nesqp->ietf_frame; - - if (cm_node->mpa_frame_rev == IETF_MPA_V1) - mpa_frame_offset = 4; - - if (cm_node->mpa_frame_rev == IETF_MPA_V1 || - cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); - } - - memcpy(mpa_v2_frame->priv_data, conn_param->private_data, - conn_param->private_data_len); - - cm_build_mpa_frame(cm_node, start_buff, &buff_len, nesqp->ietf_frame, MPA_KEY_REPLY); - nesqp->private_data_len = conn_param->private_data_len; - - /* setup our first outgoing iWarp send WQE (the IETF frame response) */ - wqe = &nesqp->hwqp.sq_vbase[0]; - - if (raddr->sin_addr.s_addr != laddr->sin_addr.s_addr) { - u64temp = (unsigned long)nesqp; - nespd = nesqp->nespd; - tagged_offset = (u64)(unsigned long)*start_buff; - ibmr = nes_reg_phys_mr(&nespd->ibpd, - nesqp->ietf_frame_pbase + mpa_frame_offset, - buff_len, IB_ACCESS_LOCAL_WRITE, - &tagged_offset); - if (IS_ERR(ibmr)) { - nes_debug(NES_DBG_CM, "Unable to register memory region" - "for lSMM for cm_node = %p \n", - cm_node); - pci_free_consistent(nesdev->pcidev, - nesqp->private_data_len + nesqp->ietf_frame_size, - nesqp->ietf_frame, nesqp->ietf_frame_pbase); - return PTR_ERR(ibmr); - } - - ibmr->pd = &nespd->ibpd; - ibmr->device = nespd->ibpd.device; - nesqp->lsmm_mr = ibmr; - - u64temp |= NES_SW_CONTEXT_ALIGN >> 1; - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, - u64temp); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_WQE_STREAMING | - NES_IWARP_SQ_WQE_WRPDU); - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = - cpu_to_le32(buff_len); - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, - (u64)(unsigned long)(*start_buff)); - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = - cpu_to_le32(buff_len); - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = ibmr->lkey; - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT | - NES_QPCONTEXT_ORDIRD_WRPDU); - } else { - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32(NES_QPCONTEXT_ORDIRD_WRPDU); - } - nesqp->skip_lsmm = 1; - - /* Cache the cm_id in the qp */ - nesqp->cm_id = cm_id; - cm_node->cm_id = cm_id; - - /* nesqp->cm_node = (void *)cm_id->provider_data; */ - cm_id->provider_data = nesqp; - nesqp->active_conn = 0; - - if (cm_node->state == NES_CM_STATE_TSA) - nes_debug(NES_DBG_CM, "Already state = TSA for cm_node=%p\n", - cm_node); - - nes_cm_init_tsa_conn(nesqp, cm_node); - - nesqp->nesqp_context->tcpPorts[0] = - cpu_to_le16(cm_node->loc_port); - nesqp->nesqp_context->tcpPorts[1] = - cpu_to_le16(cm_node->rem_port); - - nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC2_SRC_IP_SHIFT); - - nesqp->nesqp_context->arp_index_vlan |= - cpu_to_le32(nes_arp_table(nesdev, - le32_to_cpu(nesqp->nesqp_context->ip0), NULL, - NES_ARP_RESOLVE) << 16); - - nesqp->nesqp_context->ts_val_delta = cpu_to_le32( - jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW)); - - nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id); - - nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32( - ((u32)1 << NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT)); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)cm_node->ord_size); - - memset(&nes_quad, 0, sizeof(nes_quad)); - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = htonl(cm_node->rem_addr); - nes_quad.TcpPorts[0] = htons(cm_node->rem_port); - nes_quad.TcpPorts[1] = htons(cm_node->loc_port); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & adapter->hte_index_mask); - - nesqp->hte_index &= adapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - - cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node); - - nes_debug(NES_DBG_CM, "QP%u, Destination IP = 0x%08X:0x%04X, local = " - "0x%08X:0x%04X, rcv_nxt=0x%08X, snd_nxt=0x%08X, mpa + " - "private data length=%u.\n", nesqp->hwqp.qp_id, - ntohl(raddr->sin_addr.s_addr), ntohs(raddr->sin_port), - ntohl(laddr->sin_addr.s_addr), ntohs(laddr->sin_port), - le32_to_cpu(nesqp->nesqp_context->rcv_nxt), - le32_to_cpu(nesqp->nesqp_context->snd_nxt), - buff_len); - - /* notify OF layer that accept event was successful */ - cm_id->add_ref(cm_id); - nes_add_ref(&nesqp->ibqp); - - cm_event.event = IW_CM_EVENT_ESTABLISHED; - cm_event.status = 0; - cm_event.provider_data = (void *)nesqp; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - - ret = cm_id->event_handler(cm_id, &cm_event); - attr.qp_state = IB_QPS_RTS; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - if (cm_node->loopbackpartner) { - cm_node->loopbackpartner->mpa_frame_size = - nesqp->private_data_len; - /* copy entire MPA frame to our cm_node's frame */ - memcpy(cm_node->loopbackpartner->mpa_frame_buf, - conn_param->private_data, conn_param->private_data_len); - create_event(cm_node->loopbackpartner, NES_CM_EVENT_CONNECTED); - } - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - - return 0; -} - - -/** - * nes_reject - */ -int nes_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) -{ - struct nes_cm_node *cm_node; - struct nes_cm_node *loopback; - struct nes_cm_core *cm_core; - u8 *start_buff; - - atomic_inc(&cm_rejects); - cm_node = (struct nes_cm_node *)cm_id->provider_data; - loopback = cm_node->loopbackpartner; - cm_core = cm_node->cm_core; - cm_node->cm_id = cm_id; - - if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER) - return -EINVAL; - - if (loopback) { - memcpy(&loopback->mpa_frame.priv_data, pdata, pdata_len); - loopback->mpa_frame.priv_data_len = pdata_len; - loopback->mpa_frame_size = pdata_len; - } else { - start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2); - cm_node->mpa_frame_size = pdata_len; - memcpy(start_buff, pdata, pdata_len); - } - return cm_core->api->reject(cm_core, cm_node); -} - - -/** - * nes_connect - * setup and launch cm connect node - */ -int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - struct ib_qp *ibqp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_cm_info cm_info; - int apbvt_set = 0; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - - if (cm_id->remote_addr.ss_family != AF_INET) - return -ENOSYS; - ibqp = nes_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - nesqp = to_nesqp(ibqp); - if (!nesqp) - return -EINVAL; - nesvnic = to_nesvnic(nesqp->ibqp.device); - if (!nesvnic) - return -EINVAL; - nesdev = nesvnic->nesdev; - if (!nesdev) - return -EINVAL; - - if (!laddr->sin_port || !raddr->sin_port) - return -EINVAL; - - nes_debug(NES_DBG_CM, "QP%u, current IP = 0x%08X, Destination IP = " - "0x%08X:0x%04X, local = 0x%08X:0x%04X.\n", nesqp->hwqp.qp_id, - ntohl(nesvnic->local_ipaddr), ntohl(raddr->sin_addr.s_addr), - ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr), - ntohs(laddr->sin_port)); - - atomic_inc(&cm_connects); - nesqp->active_conn = 1; - - /* cache the cm_id in the qp */ - nesqp->cm_id = cm_id; - cm_id->provider_data = nesqp; - nesqp->private_data_len = conn_param->private_data_len; - - nes_debug(NES_DBG_CM, "requested ord = 0x%08X.\n", (u32)conn_param->ord); - nes_debug(NES_DBG_CM, "mpa private data len =%u\n", - conn_param->private_data_len); - - /* set up the connection params for the node */ - cm_info.loc_addr = ntohl(laddr->sin_addr.s_addr); - cm_info.loc_port = ntohs(laddr->sin_port); - cm_info.rem_addr = ntohl(raddr->sin_addr.s_addr); - cm_info.rem_port = ntohs(raddr->sin_port); - cm_info.cm_id = cm_id; - cm_info.conn_type = NES_CM_IWARP_CONN_TYPE; - - if (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr) { - nes_manage_apbvt(nesvnic, cm_info.loc_port, - PCI_FUNC(nesdev->pcidev->devfn), - NES_MANAGE_APBVT_ADD); - apbvt_set = 1; - } - - cm_id->add_ref(cm_id); - - /* create a connect CM node connection */ - cm_node = g_cm_core->api->connect(g_cm_core, nesvnic, - conn_param->private_data_len, (void *)conn_param->private_data, - &cm_info); - if (!cm_node) { - if (apbvt_set) - nes_manage_apbvt(nesvnic, cm_info.loc_port, - PCI_FUNC(nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - - nes_debug(NES_DBG_NLMSG, "Delete loc_port = %04X\n", - cm_info.loc_port); - cm_id->rem_ref(cm_id); - return -ENOMEM; - } - - record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); - if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO && - cm_node->ord_size == 0) - cm_node->ord_size = 1; - - cm_node->apbvt_set = apbvt_set; - cm_node->tos = cm_id->tos; - nesqp->cm_node = cm_node; - cm_node->nesqp = nesqp; - nes_add_ref(&nesqp->ibqp); - - return 0; -} - - -/** - * nes_create_listen - */ -int nes_create_listen(struct iw_cm_id *cm_id, int backlog) -{ - struct nes_vnic *nesvnic; - struct nes_cm_listener *cm_node; - struct nes_cm_info cm_info; - int err; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - - nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n", - cm_id, ntohs(laddr->sin_port)); - - if (cm_id->m_local_addr.ss_family != AF_INET) - return -ENOSYS; - nesvnic = to_nesvnic(cm_id->device); - if (!nesvnic) - return -EINVAL; - - nes_debug(NES_DBG_CM, "nesvnic=%p, netdev=%p, %s\n", - nesvnic, nesvnic->netdev, nesvnic->netdev->name); - - nes_debug(NES_DBG_CM, "nesvnic->local_ipaddr=0x%08x, sin_addr.s_addr=0x%08x\n", - nesvnic->local_ipaddr, laddr->sin_addr.s_addr); - - /* setup listen params in our api call struct */ - cm_info.loc_addr = ntohl(nesvnic->local_ipaddr); - cm_info.loc_port = ntohs(laddr->sin_port); - cm_info.backlog = backlog; - cm_info.cm_id = cm_id; - - cm_info.conn_type = NES_CM_IWARP_CONN_TYPE; - - cm_node = g_cm_core->api->listen(g_cm_core, nesvnic, &cm_info); - if (!cm_node) { - printk(KERN_ERR "%s[%u] Error returned from listen API call\n", - __func__, __LINE__); - return -ENOMEM; - } - - cm_id->provider_data = cm_node; - cm_node->tos = cm_id->tos; - - if (!cm_node->reused_node) { - err = nes_manage_apbvt(nesvnic, cm_node->loc_port, - PCI_FUNC(nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_ADD); - if (err) { - printk(KERN_ERR "nes_manage_apbvt call returned %d.\n", - err); - g_cm_core->api->stop_listener(g_cm_core, (void *)cm_node); - return err; - } - atomic_inc(&cm_listens_created); - } - - cm_id->add_ref(cm_id); - cm_id->provider_data = (void *)cm_node; - - - return 0; -} - - -/** - * nes_destroy_listen - */ -int nes_destroy_listen(struct iw_cm_id *cm_id) -{ - if (cm_id->provider_data) - g_cm_core->api->stop_listener(g_cm_core, cm_id->provider_data); - else - nes_debug(NES_DBG_CM, "cm_id->provider_data was NULL\n"); - - cm_id->rem_ref(cm_id); - - return 0; -} - - -/** - * nes_cm_recv - */ -int nes_cm_recv(struct sk_buff *skb, struct net_device *netdevice) -{ - int rc = 0; - - cm_packets_received++; - if ((g_cm_core) && (g_cm_core->api)) - rc = g_cm_core->api->recv_pkt(g_cm_core, netdev_priv(netdevice), skb); - else - nes_debug(NES_DBG_CM, "Unable to process packet for CM," - " cm is not setup properly.\n"); - - return rc; -} - - -/** - * nes_cm_start - * Start and init a cm core module - */ -int nes_cm_start(void) -{ - nes_debug(NES_DBG_CM, "\n"); - /* create the primary CM core, pass this handle to subsequent core inits */ - g_cm_core = nes_cm_alloc_core(); - if (g_cm_core) - return 0; - else - return -ENOMEM; -} - - -/** - * nes_cm_stop - * stop and dealloc all cm core instances - */ -int nes_cm_stop(void) -{ - g_cm_core->api->destroy_cm_core(g_cm_core); - return 0; -} - - -/** - * cm_event_connected - * handle a connected event, setup QPs and HW - */ -static void cm_event_connected(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_adapter *nesadapter; - struct ib_qp_attr attr; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_v4_quad nes_quad; - u32 crc_value; - int ret; - struct sockaddr_in *laddr; - struct sockaddr_in *raddr; - struct sockaddr_in *cm_event_laddr; - - /* get all our handles */ - cm_node = event->cm_node; - cm_id = cm_node->cm_id; - nes_debug(NES_DBG_CM, "cm_event_connected - %p - cm_id = %p\n", cm_node, cm_id); - nesqp = (struct nes_qp *)cm_id->provider_data; - nesvnic = to_nesvnic(nesqp->ibqp.device); - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - cm_event_laddr = (struct sockaddr_in *)&cm_event.local_addr; - - if (nesqp->destroyed) - return; - atomic_inc(&cm_connecteds); - nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on" - " local port 0x%04X. jiffies = %lu.\n", - nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr), - ntohs(raddr->sin_port), ntohs(laddr->sin_port), jiffies); - - nes_cm_init_tsa_conn(nesqp, cm_node); - - /* set the QP tsa context */ - nesqp->nesqp_context->tcpPorts[0] = - cpu_to_le16(cm_node->loc_port); - nesqp->nesqp_context->tcpPorts[1] = - cpu_to_le16(cm_node->rem_port); - nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC2_SRC_IP_SHIFT); - nesqp->nesqp_context->arp_index_vlan |= cpu_to_le32( - nes_arp_table(nesdev, - le32_to_cpu(nesqp->nesqp_context->ip0), - NULL, NES_ARP_RESOLVE) << 16); - nesqp->nesqp_context->ts_val_delta = cpu_to_le32( - jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW)); - nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)1 << - NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)cm_node->ord_size); - - /* Adjust tail for not having a LSMM */ - /*nesqp->hwqp.sq_tail = 1;*/ - - build_rdma0_msg(cm_node, &nesqp); - - nes_write32(nesdev->regs + NES_WQE_ALLOC, - (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id); - - memset(&nes_quad, 0, sizeof(nes_quad)); - - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = htonl(cm_node->rem_addr); - nes_quad.TcpPorts[0] = htons(cm_node->rem_port); - nes_quad.TcpPorts[1] = htons(cm_node->loc_port); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, After CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask); - - nesqp->hte_index &= nesadapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - - nesqp->ietf_frame = &cm_node->mpa_frame; - nesqp->private_data_len = (u8)cm_node->mpa_frame_size; - cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node); - - /* notify OF layer we successfully created the requested connection */ - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = laddr->sin_port; - cm_event.remote_addr = cm_id->m_remote_addr; - - cm_event.private_data = (void *)event->cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)event->cm_node->mpa_frame_size; - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - ret = cm_id->event_handler(cm_id, &cm_event); - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - attr.qp_state = IB_QPS_RTS; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - - nes_debug(NES_DBG_CM, "Exiting connect thread for QP%u. jiffies = " - "%lu\n", nesqp->hwqp.qp_id, jiffies); - - return; -} - - -/** - * cm_event_connect_error - */ -static void cm_event_connect_error(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - /* struct nes_cm_info cm_info; */ - int ret; - - if (!event->cm_node) - return; - - cm_id = event->cm_node->cm_id; - if (!cm_id) - return; - - nes_debug(NES_DBG_CM, "cm_node=%p, cm_id=%p\n", event->cm_node, cm_id); - nesqp = cm_id->provider_data; - - if (!nesqp) - return; - - /* notify OF layer about this connection error event */ - /* cm_id->rem_ref(cm_id); */ - nesqp->cm_id = NULL; - cm_id->provider_data = NULL; - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ECONNRESET; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - -#ifdef CONFIG_INFINIBAND_NES_DEBUG - { - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - nes_debug(NES_DBG_CM, "call CM_EVENT REJECTED, local_addr=%08x, remote_addr=%08x\n", - cm_event_laddr->sin_addr.s_addr, cm_event_raddr->sin_addr.s_addr); - } -#endif - - ret = cm_id->event_handler(cm_id, &cm_event); - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - cm_id->rem_ref(cm_id); - - rem_ref_cm_node(event->cm_node->cm_core, event->cm_node); - return; -} - - -/** - * cm_event_reset - */ -static void cm_event_reset(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - /* struct nes_cm_info cm_info; */ - int ret; - - if (!event->cm_node) - return; - - if (!event->cm_node->cm_id) - return; - - cm_id = event->cm_node->cm_id; - - nes_debug(NES_DBG_CM, "%p - cm_id = %p\n", event->cm_node, cm_id); - nesqp = cm_id->provider_data; - if (!nesqp) - return; - - nesqp->cm_id = NULL; - /* cm_id->provider_data = NULL; */ - cm_event.event = IW_CM_EVENT_DISCONNECT; - cm_event.status = -ECONNRESET; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - cm_id->add_ref(cm_id); - ret = cm_id->event_handler(cm_id, &cm_event); - atomic_inc(&cm_closes); - cm_event.event = IW_CM_EVENT_CLOSE; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - nes_debug(NES_DBG_CM, "NODE %p Generating CLOSE\n", event->cm_node); - ret = cm_id->event_handler(cm_id, &cm_event); - - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - - /* notify OF layer about this connection error event */ - cm_id->rem_ref(cm_id); - - return; -} - - -/** - * cm_event_mpa_req - */ -static void cm_event_mpa_req(struct nes_cm_event *event) -{ - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - int ret; - struct nes_cm_node *cm_node; - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - - cm_node = event->cm_node; - if (!cm_node) - return; - cm_id = cm_node->cm_id; - - atomic_inc(&cm_connect_reqs); - nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n", - cm_node, cm_id, jiffies); - - cm_event.event = IW_CM_EVENT_CONNECT_REQUEST; - cm_event.status = 0; - cm_event.provider_data = (void *)cm_node; - - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = htons(event->cm_info.loc_port); - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - - cm_event_raddr->sin_family = AF_INET; - cm_event_raddr->sin_port = htons(event->cm_info.rem_port); - cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr); - cm_event.private_data = cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)cm_node->mpa_frame_size; - if (cm_node->mpa_frame_rev == IETF_MPA_V1) { - cm_event.ird = NES_MAX_IRD; - cm_event.ord = NES_MAX_ORD; - } else { - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - } - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n", - __func__, __LINE__, ret); - return; -} - - -static void cm_event_mpa_reject(struct nes_cm_event *event) -{ - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_cm_node *cm_node; - int ret; - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - - cm_node = event->cm_node; - if (!cm_node) - return; - cm_id = cm_node->cm_id; - - atomic_inc(&cm_connect_reqs); - nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n", - cm_node, cm_id, jiffies); - - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ECONNREFUSED; - cm_event.provider_data = cm_id->provider_data; - - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = htons(event->cm_info.loc_port); - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - - cm_event_raddr->sin_family = AF_INET; - cm_event_raddr->sin_port = htons(event->cm_info.rem_port); - cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr); - - cm_event.private_data = cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)cm_node->mpa_frame_size; - - nes_debug(NES_DBG_CM, "call CM_EVENT_MPA_REJECTED, local_addr=%08x, " - "remove_addr=%08x\n", - cm_event_laddr->sin_addr.s_addr, - cm_event_raddr->sin_addr.s_addr); - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n", - __func__, __LINE__, ret); - - return; -} - - -static void nes_cm_event_handler(struct work_struct *); - -/** - * nes_cm_post_event - * post an event to the cm event handler - */ -static int nes_cm_post_event(struct nes_cm_event *event) -{ - atomic_inc(&event->cm_node->cm_core->events_posted); - add_ref_cm_node(event->cm_node); - event->cm_info.cm_id->add_ref(event->cm_info.cm_id); - INIT_WORK(&event->event_work, nes_cm_event_handler); - nes_debug(NES_DBG_CM, "cm_node=%p queue_work, event=%p\n", - event->cm_node, event); - - queue_work(event->cm_node->cm_core->event_wq, &event->event_work); - - nes_debug(NES_DBG_CM, "Exit\n"); - return 0; -} - - -/** - * nes_cm_event_handler - * worker function to handle cm events - * will free instance of nes_cm_event - */ -static void nes_cm_event_handler(struct work_struct *work) -{ - struct nes_cm_event *event = container_of(work, struct nes_cm_event, - event_work); - struct nes_cm_core *cm_core; - - if ((!event) || (!event->cm_node) || (!event->cm_node->cm_core)) - return; - - cm_core = event->cm_node->cm_core; - nes_debug(NES_DBG_CM, "event=%p, event->type=%u, events posted=%u\n", - event, event->type, atomic_read(&cm_core->events_posted)); - - switch (event->type) { - case NES_CM_EVENT_MPA_REQ: - cm_event_mpa_req(event); - nes_debug(NES_DBG_CM, "cm_node=%p CM Event: MPA REQUEST\n", - event->cm_node); - break; - case NES_CM_EVENT_RESET: - nes_debug(NES_DBG_CM, "cm_node = %p CM Event: RESET\n", - event->cm_node); - cm_event_reset(event); - break; - case NES_CM_EVENT_CONNECTED: - if ((!event->cm_node->cm_id) || - (event->cm_node->state != NES_CM_STATE_TSA)) - break; - cm_event_connected(event); - nes_debug(NES_DBG_CM, "CM Event: CONNECTED\n"); - break; - case NES_CM_EVENT_MPA_REJECT: - if ((!event->cm_node->cm_id) || - (event->cm_node->state == NES_CM_STATE_TSA)) - break; - cm_event_mpa_reject(event); - nes_debug(NES_DBG_CM, "CM Event: REJECT\n"); - break; - - case NES_CM_EVENT_ABORTED: - if ((!event->cm_node->cm_id) || - (event->cm_node->state == NES_CM_STATE_TSA)) - break; - cm_event_connect_error(event); - nes_debug(NES_DBG_CM, "CM Event: ABORTED\n"); - break; - case NES_CM_EVENT_DROPPED_PKT: - nes_debug(NES_DBG_CM, "CM Event: DROPPED PKT\n"); - break; - default: - nes_debug(NES_DBG_CM, "CM Event: UNKNOWN EVENT TYPE\n"); - break; - } - - atomic_dec(&cm_core->events_posted); - event->cm_info.cm_id->rem_ref(event->cm_info.cm_id); - rem_ref_cm_node(cm_core, event->cm_node); - kfree(event); - - return; -} diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h deleted file mode 100644 index b9cc02b4e8d5..000000000000 --- a/drivers/infiniband/hw/nes/nes_cm.h +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef NES_CM_H -#define NES_CM_H - -#define QUEUE_EVENTS - -#define NES_MANAGE_APBVT_DEL 0 -#define NES_MANAGE_APBVT_ADD 1 - -#define NES_MPA_REQUEST_ACCEPT 1 -#define NES_MPA_REQUEST_REJECT 2 - -/* IETF MPA -- defines, enums, structs */ -#define IEFT_MPA_KEY_REQ "MPA ID Req Frame" -#define IEFT_MPA_KEY_REP "MPA ID Rep Frame" -#define IETF_MPA_KEY_SIZE 16 -#define IETF_MPA_VERSION 1 -#define IETF_MAX_PRIV_DATA_LEN 512 -#define IETF_MPA_FRAME_SIZE 20 -#define IETF_RTR_MSG_SIZE 4 -#define IETF_MPA_V2_FLAG 0x10 - -/* IETF RTR MSG Fields */ -#define IETF_PEER_TO_PEER 0x8000 -#define IETF_FLPDU_ZERO_LEN 0x4000 -#define IETF_RDMA0_WRITE 0x8000 -#define IETF_RDMA0_READ 0x4000 -#define IETF_NO_IRD_ORD 0x3FFF -#define NES_MAX_IRD 0x40 -#define NES_MAX_ORD 0x7F - -enum ietf_mpa_flags { - IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */ - IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */ - IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */ -}; - -struct ietf_mpa_v1 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - u8 priv_data[0]; -}; - -#define ietf_mpa_req_resp_frame ietf_mpa_frame - -struct ietf_rtr_msg { - __be16 ctrl_ird; - __be16 ctrl_ord; -}; - -struct ietf_mpa_v2 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - struct ietf_rtr_msg rtr_msg; - u8 priv_data[0]; -}; - -struct nes_v4_quad { - u32 rsvd0; - __le32 DstIpAdrIndex; /* Only most significant 5 bits are valid */ - __be32 SrcIpadr; - __be16 TcpPorts[2]; /* src is low, dest is high */ -}; - -struct nes_cm_node; -enum nes_timer_type { - NES_TIMER_TYPE_SEND, - NES_TIMER_TYPE_RECV, - NES_TIMER_NODE_CLEANUP, - NES_TIMER_TYPE_CLOSE, -}; - -#define NES_PASSIVE_STATE_INDICATED 0 -#define NES_DO_NOT_SEND_RESET_EVENT 1 -#define NES_SEND_RESET_EVENT 2 - -#define MAX_NES_IFS 4 - -#define SET_ACK 1 -#define SET_SYN 2 -#define SET_FIN 4 -#define SET_RST 8 - -#define TCP_OPTIONS_PADDING 3 - -struct option_base { - u8 optionnum; - u8 length; -}; - -enum option_numbers { - OPTION_NUMBER_END, - OPTION_NUMBER_NONE, - OPTION_NUMBER_MSS, - OPTION_NUMBER_WINDOW_SCALE, - OPTION_NUMBER_SACK_PERM, - OPTION_NUMBER_SACK, - OPTION_NUMBER_WRITE0 = 0xbc -}; - -struct option_mss { - u8 optionnum; - u8 length; - __be16 mss; -}; - -struct option_windowscale { - u8 optionnum; - u8 length; - u8 shiftcount; -}; - -union all_known_options { - char as_end; - struct option_base as_base; - struct option_mss as_mss; - struct option_windowscale as_windowscale; -}; - -struct nes_timer_entry { - struct list_head list; - unsigned long timetosend; /* jiffies */ - struct sk_buff *skb; - u32 type; - u32 retrycount; - u32 retranscount; - u32 context; - u32 seq_num; - u32 send_retrans; - int close_when_complete; - struct net_device *netdev; -}; - -#define NES_DEFAULT_RETRYS 64 -#define NES_DEFAULT_RETRANS 8 -#ifdef CONFIG_INFINIBAND_NES_DEBUG -#define NES_RETRY_TIMEOUT (1000*HZ/1000) -#else -#define NES_RETRY_TIMEOUT (3000*HZ/1000) -#endif -#define NES_SHORT_TIME (10) -#define NES_LONG_TIME (2000*HZ/1000) -#define NES_MAX_TIMEOUT ((unsigned long) (12*HZ)) - -#define NES_CM_HASHTABLE_SIZE 1024 -#define NES_CM_TCP_TIMER_INTERVAL 3000 -#define NES_CM_DEFAULT_MTU 1540 -#define NES_CM_DEFAULT_FRAME_CNT 10 -#define NES_CM_THREAD_STACK_SIZE 256 -#define NES_CM_DEFAULT_RCV_WND 64240 // before we know that window scaling is allowed -#define NES_CM_DEFAULT_RCV_WND_SCALED 256960 // after we know that window scaling is allowed -#define NES_CM_DEFAULT_RCV_WND_SCALE 2 -#define NES_CM_DEFAULT_FREE_PKTS 0x000A -#define NES_CM_FREE_PKT_LO_WATERMARK 2 - -#define NES_CM_DEFAULT_MSS 536 - -#define NES_CM_DEF_SEQ 0x159bf75f -#define NES_CM_DEF_LOCAL_ID 0x3b47 - -#define NES_CM_DEF_SEQ2 0x18ed5740 -#define NES_CM_DEF_LOCAL_ID2 0xb807 -#define MAX_CM_BUFFER (IETF_MPA_FRAME_SIZE + IETF_RTR_MSG_SIZE + IETF_MAX_PRIV_DATA_LEN) - -typedef u32 nes_addr_t; - -#define nes_cm_tsa_context nes_qp_context - -struct nes_qp; - -/* cm node transition states */ -enum nes_cm_node_state { - NES_CM_STATE_UNKNOWN, - NES_CM_STATE_INITED, - NES_CM_STATE_LISTENING, - NES_CM_STATE_SYN_RCVD, - NES_CM_STATE_SYN_SENT, - NES_CM_STATE_ONE_SIDE_ESTABLISHED, - NES_CM_STATE_ESTABLISHED, - NES_CM_STATE_ACCEPTING, - NES_CM_STATE_MPAREQ_SENT, - NES_CM_STATE_MPAREQ_RCVD, - NES_CM_STATE_MPAREJ_RCVD, - NES_CM_STATE_TSA, - NES_CM_STATE_FIN_WAIT1, - NES_CM_STATE_FIN_WAIT2, - NES_CM_STATE_CLOSE_WAIT, - NES_CM_STATE_TIME_WAIT, - NES_CM_STATE_LAST_ACK, - NES_CM_STATE_CLOSING, - NES_CM_STATE_LISTENER_DESTROYED, - NES_CM_STATE_CLOSED -}; - -enum mpa_frame_version { - IETF_MPA_V1 = 1, - IETF_MPA_V2 = 2 -}; - -enum mpa_frame_key { - MPA_KEY_REQUEST, - MPA_KEY_REPLY -}; - -enum send_rdma0 { - SEND_RDMA_READ_ZERO = 1, - SEND_RDMA_WRITE_ZERO = 2 -}; - -enum nes_tcpip_pkt_type { - NES_PKT_TYPE_UNKNOWN, - NES_PKT_TYPE_SYN, - NES_PKT_TYPE_SYNACK, - NES_PKT_TYPE_ACK, - NES_PKT_TYPE_FIN, - NES_PKT_TYPE_RST -}; - - -/* type of nes connection */ -enum nes_cm_conn_type { - NES_CM_IWARP_CONN_TYPE, -}; - -/* CM context params */ -struct nes_cm_tcp_context { - u8 client; - - u32 loc_seq_num; - u32 loc_ack_num; - u32 rem_ack_num; - u32 rcv_nxt; - - u32 loc_id; - u32 rem_id; - - u32 snd_wnd; - u32 max_snd_wnd; - - u32 rcv_wnd; - u32 mss; - u8 snd_wscale; - u8 rcv_wscale; - - struct nes_cm_tsa_context tsa_cntxt; -}; - - -enum nes_cm_listener_state { - NES_CM_LISTENER_PASSIVE_STATE = 1, - NES_CM_LISTENER_ACTIVE_STATE = 2, - NES_CM_LISTENER_EITHER_STATE = 3 -}; - -struct nes_cm_listener { - struct list_head list; - struct nes_cm_core *cm_core; - u8 loc_mac[ETH_ALEN]; - nes_addr_t loc_addr; - u16 loc_port; - struct iw_cm_id *cm_id; - enum nes_cm_conn_type conn_type; - atomic_t ref_count; - struct nes_vnic *nesvnic; - atomic_t pend_accepts_cnt; - int backlog; - enum nes_cm_listener_state listener_state; - u32 reused_node; - u8 tos; -}; - -/* per connection node and node state information */ -struct nes_cm_node { - nes_addr_t loc_addr, rem_addr; - u16 loc_port, rem_port; - - u8 loc_mac[ETH_ALEN]; - u8 rem_mac[ETH_ALEN]; - - enum nes_cm_node_state state; - struct nes_cm_tcp_context tcp_cntxt; - struct nes_cm_core *cm_core; - struct sk_buff_head resend_list; - atomic_t ref_count; - struct net_device *netdev; - - struct nes_cm_node *loopbackpartner; - - struct nes_timer_entry *send_entry; - struct nes_timer_entry *recv_entry; - spinlock_t retrans_list_lock; - enum send_rdma0 send_rdma0_op; - - union { - struct ietf_mpa_v1 mpa_frame; - struct ietf_mpa_v2 mpa_v2_frame; - u8 mpa_frame_buf[MAX_CM_BUFFER]; - }; - enum mpa_frame_version mpa_frame_rev; - u16 ird_size; - u16 ord_size; - u16 mpav2_ird_ord; - - u16 mpa_frame_size; - struct iw_cm_id *cm_id; - struct list_head list; - bool accelerated; - struct nes_cm_listener *listener; - enum nes_cm_conn_type conn_type; - struct nes_vnic *nesvnic; - int apbvt_set; - int accept_pend; - struct list_head timer_entry; - struct list_head reset_entry; - struct nes_qp *nesqp; - atomic_t passive_state; - u8 tos; -}; - -/* structure for client or CM to fill when making CM api calls. */ -/* - only need to set relevant data, based on op. */ -struct nes_cm_info { - union { - struct iw_cm_id *cm_id; - struct net_device *netdev; - }; - - u16 loc_port; - u16 rem_port; - nes_addr_t loc_addr; - nes_addr_t rem_addr; - enum nes_cm_conn_type conn_type; - int backlog; -}; - -/* CM event codes */ -enum nes_cm_event_type { - NES_CM_EVENT_UNKNOWN, - NES_CM_EVENT_ESTABLISHED, - NES_CM_EVENT_MPA_REQ, - NES_CM_EVENT_MPA_CONNECT, - NES_CM_EVENT_MPA_ACCEPT, - NES_CM_EVENT_MPA_REJECT, - NES_CM_EVENT_MPA_ESTABLISHED, - NES_CM_EVENT_CONNECTED, - NES_CM_EVENT_CLOSED, - NES_CM_EVENT_RESET, - NES_CM_EVENT_DROPPED_PKT, - NES_CM_EVENT_CLOSE_IMMED, - NES_CM_EVENT_CLOSE_HARD, - NES_CM_EVENT_CLOSE_CLEAN, - NES_CM_EVENT_ABORTED, - NES_CM_EVENT_SEND_FIRST -}; - -/* event to post to CM event handler */ -struct nes_cm_event { - enum nes_cm_event_type type; - - struct nes_cm_info cm_info; - struct work_struct event_work; - struct nes_cm_node *cm_node; -}; - -struct nes_cm_core { - enum nes_cm_node_state state; - - atomic_t listen_node_cnt; - struct nes_cm_node listen_list; - spinlock_t listen_list_lock; - - u32 mtu; - u32 free_tx_pkt_max; - u32 rx_pkt_posted; - atomic_t ht_node_cnt; - struct list_head connected_nodes; - /* struct list_head hashtable[NES_CM_HASHTABLE_SIZE]; */ - spinlock_t ht_lock; - - struct timer_list tcp_timer; - - const struct nes_cm_ops *api; - - int (*post_event)(struct nes_cm_event *event); - atomic_t events_posted; - struct workqueue_struct *event_wq; - struct workqueue_struct *disconn_wq; - - atomic_t node_cnt; - u64 aborted_connects; - u32 options; - - struct nes_cm_node *current_listen_node; -}; - - -#define NES_CM_SET_PKT_SIZE (1 << 1) -#define NES_CM_SET_FREE_PKT_Q_SIZE (1 << 2) - -/* CM ops/API for client interface */ -struct nes_cm_ops { - int (*accelerated)(struct nes_cm_core *, struct nes_cm_node *); - struct nes_cm_listener * (*listen)(struct nes_cm_core *, struct nes_vnic *, - struct nes_cm_info *); - int (*stop_listener)(struct nes_cm_core *, struct nes_cm_listener *); - struct nes_cm_node * (*connect)(struct nes_cm_core *, - struct nes_vnic *, u16, void *, - struct nes_cm_info *); - int (*close)(struct nes_cm_core *, struct nes_cm_node *); - int (*accept)(struct nes_cm_core *, struct nes_cm_node *); - int (*reject)(struct nes_cm_core *, struct nes_cm_node *); - int (*recv_pkt)(struct nes_cm_core *, struct nes_vnic *, - struct sk_buff *); - int (*destroy_cm_core)(struct nes_cm_core *); - int (*get)(struct nes_cm_core *); - int (*set)(struct nes_cm_core *, u32, u32); -}; - -int schedule_nes_timer(struct nes_cm_node *, struct sk_buff *, - enum nes_timer_type, int, int); - -int nes_accept(struct iw_cm_id *, struct iw_cm_conn_param *); -int nes_reject(struct iw_cm_id *, const void *, u8); -int nes_connect(struct iw_cm_id *, struct iw_cm_conn_param *); -int nes_create_listen(struct iw_cm_id *, int); -int nes_destroy_listen(struct iw_cm_id *); - -int nes_cm_recv(struct sk_buff *, struct net_device *); -int nes_cm_start(void); -int nes_cm_stop(void); -int nes_add_ref_cm_node(struct nes_cm_node *cm_node); -int nes_rem_ref_cm_node(struct nes_cm_node *cm_node); - -#endif /* NES_CM_H */ diff --git a/drivers/infiniband/hw/nes/nes_context.h b/drivers/infiniband/hw/nes/nes_context.h deleted file mode 100644 index a69eef16d72d..000000000000 --- a/drivers/infiniband/hw/nes/nes_context.h +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef NES_CONTEXT_H -#define NES_CONTEXT_H - -struct nes_qp_context { - __le32 misc; - __le32 cqs; - __le32 sq_addr_low; - __le32 sq_addr_high; - __le32 rq_addr_low; - __le32 rq_addr_high; - __le32 misc2; - __le16 tcpPorts[2]; - __le32 ip0; - __le32 ip1; - __le32 ip2; - __le32 ip3; - __le32 mss; - __le32 arp_index_vlan; - __le32 tcp_state_flow_label; - __le32 pd_index_wscale; - __le32 keepalive; - u32 ts_recent; - u32 ts_age; - __le32 snd_nxt; - __le32 snd_wnd; - __le32 rcv_nxt; - __le32 rcv_wnd; - __le32 snd_max; - __le32 snd_una; - u32 srtt; - __le32 rttvar; - __le32 ssthresh; - __le32 cwnd; - __le32 snd_wl1; - __le32 snd_wl2; - __le32 max_snd_wnd; - __le32 ts_val_delta; - u32 retransmit; - u32 probe_cnt; - u32 hte_index; - __le32 q2_addr_low; - __le32 q2_addr_high; - __le32 ird_index; - u32 Rsvd3; - __le32 ird_ord_sizes; - u32 mrkr_offset; - __le32 aeq_token_low; - __le32 aeq_token_high; -}; - -/* QP Context Misc Field */ - -#define NES_QPCONTEXT_MISC_IWARP_VER_MASK 0x00000003 -#define NES_QPCONTEXT_MISC_IWARP_VER_SHIFT 0 -#define NES_QPCONTEXT_MISC_EFB_SIZE_MASK 0x000000C0 -#define NES_QPCONTEXT_MISC_EFB_SIZE_SHIFT 6 -#define NES_QPCONTEXT_MISC_RQ_SIZE_MASK 0x00000300 -#define NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT 8 -#define NES_QPCONTEXT_MISC_SQ_SIZE_MASK 0x00000c00 -#define NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT 10 -#define NES_QPCONTEXT_MISC_PCI_FCN_MASK 0x00007000 -#define NES_QPCONTEXT_MISC_PCI_FCN_SHIFT 12 -#define NES_QPCONTEXT_MISC_DUP_ACKS_MASK 0x00070000 -#define NES_QPCONTEXT_MISC_DUP_ACKS_SHIFT 16 - -enum nes_qp_context_misc_bits { - NES_QPCONTEXT_MISC_RX_WQE_SIZE = 0x00000004, - NES_QPCONTEXT_MISC_IPV4 = 0x00000008, - NES_QPCONTEXT_MISC_DO_NOT_FRAG = 0x00000010, - NES_QPCONTEXT_MISC_INSERT_VLAN = 0x00000020, - NES_QPCONTEXT_MISC_DROS = 0x00008000, - NES_QPCONTEXT_MISC_WSCALE = 0x00080000, - NES_QPCONTEXT_MISC_KEEPALIVE = 0x00100000, - NES_QPCONTEXT_MISC_TIMESTAMP = 0x00200000, - NES_QPCONTEXT_MISC_SACK = 0x00400000, - NES_QPCONTEXT_MISC_RDMA_WRITE_EN = 0x00800000, - NES_QPCONTEXT_MISC_RDMA_READ_EN = 0x01000000, - NES_QPCONTEXT_MISC_WBIND_EN = 0x10000000, - NES_QPCONTEXT_MISC_FAST_REGISTER_EN = 0x20000000, - NES_QPCONTEXT_MISC_PRIV_EN = 0x40000000, - NES_QPCONTEXT_MISC_NO_NAGLE = 0x80000000 -}; - -enum nes_qp_acc_wq_sizes { - HCONTEXT_TSA_WQ_SIZE_4 = 0, - HCONTEXT_TSA_WQ_SIZE_32 = 1, - HCONTEXT_TSA_WQ_SIZE_128 = 2, - HCONTEXT_TSA_WQ_SIZE_512 = 3 -}; - -/* QP Context Misc2 Fields */ -#define NES_QPCONTEXT_MISC2_TTL_MASK 0x000000ff -#define NES_QPCONTEXT_MISC2_TTL_SHIFT 0 -#define NES_QPCONTEXT_MISC2_HOP_LIMIT_MASK 0x000000ff -#define NES_QPCONTEXT_MISC2_HOP_LIMIT_SHIFT 0 -#define NES_QPCONTEXT_MISC2_LIMIT_MASK 0x00000300 -#define NES_QPCONTEXT_MISC2_LIMIT_SHIFT 8 -#define NES_QPCONTEXT_MISC2_NIC_INDEX_MASK 0x0000fc00 -#define NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT 10 -#define NES_QPCONTEXT_MISC2_SRC_IP_MASK 0x001f0000 -#define NES_QPCONTEXT_MISC2_SRC_IP_SHIFT 16 -#define NES_QPCONTEXT_MISC2_TOS_MASK 0xff000000 -#define NES_QPCONTEXT_MISC2_TOS_SHIFT 24 -#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_MASK 0xff000000 -#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_SHIFT 24 - -/* QP Context Tcp State/Flow Label Fields */ -#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_MASK 0x000fffff -#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_SHIFT 0 -#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_MASK 0xf0000000 -#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT 28 - -enum nes_qp_tcp_state { - NES_QPCONTEXT_TCPSTATE_CLOSED = 1, - NES_QPCONTEXT_TCPSTATE_EST = 5, - NES_QPCONTEXT_TCPSTATE_TIME_WAIT = 11, -}; - -/* QP Context PD Index/wscale Fields */ -#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK 0x0000000f -#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT 0 -#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK 0x00000f00 -#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT 8 -#define NES_QPCONTEXT_PDWSCALE_PDINDEX_MASK 0xffff0000 -#define NES_QPCONTEXT_PDWSCALE_PDINDEX_SHIFT 16 - -/* QP Context Keepalive Fields */ -#define NES_QPCONTEXT_KEEPALIVE_DELTA_MASK 0x0000ffff -#define NES_QPCONTEXT_KEEPALIVE_DELTA_SHIFT 0 -#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_MASK 0x00ff0000 -#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_SHIFT 16 -#define NES_QPCONTEXT_KEEPALIVE_INTV_MASK 0xff000000 -#define NES_QPCONTEXT_KEEPALIVE_INTV_SHIFT 24 - -/* QP Context ORD/IRD Fields */ -#define NES_QPCONTEXT_ORDIRD_ORDSIZE_MASK 0x0000007f -#define NES_QPCONTEXT_ORDIRD_ORDSIZE_SHIFT 0 -#define NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK 0x00030000 -#define NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT 16 -#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_MASK 0x30000000 -#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT 28 - -enum nes_ord_ird_bits { - NES_QPCONTEXT_ORDIRD_WRPDU = 0x02000000, - NES_QPCONTEXT_ORDIRD_LSMM_PRESENT = 0x04000000, - NES_QPCONTEXT_ORDIRD_ALSMM = 0x08000000, - NES_QPCONTEXT_ORDIRD_AAH = 0x40000000, - NES_QPCONTEXT_ORDIRD_RNMC = 0x80000000 -}; - -enum nes_iwarp_qp_state { - NES_QPCONTEXT_IWARP_STATE_NONEXIST = 0, - NES_QPCONTEXT_IWARP_STATE_IDLE = 1, - NES_QPCONTEXT_IWARP_STATE_RTS = 2, - NES_QPCONTEXT_IWARP_STATE_CLOSING = 3, - NES_QPCONTEXT_IWARP_STATE_TERMINATE = 5, - NES_QPCONTEXT_IWARP_STATE_ERROR = 6 -}; - - -#endif /* NES_CONTEXT_H */ diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c deleted file mode 100644 index 5517e392bc01..000000000000 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ /dev/null @@ -1,3887 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include - -#include "nes.h" - -static int wide_ppm_offset; -module_param(wide_ppm_offset, int, 0644); -MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm"); - -static u32 crit_err_count; -u32 int_mod_timer_init; -u32 int_mod_cq_depth_256; -u32 int_mod_cq_depth_128; -u32 int_mod_cq_depth_32; -u32 int_mod_cq_depth_24; -u32 int_mod_cq_depth_16; -u32 int_mod_cq_depth_4; -u32 int_mod_cq_depth_1; -static const u8 nes_max_critical_error_count = 100; -#include "nes_cm.h" - -static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq); -static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count); -static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, - struct nes_adapter *nesadapter, u8 OneG_Mode); -static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq); -static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq); -static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq); -static void nes_process_iwarp_aeqe(struct nes_device *nesdev, - struct nes_hw_aeqe *aeqe); -static void process_critical_error(struct nes_device *nesdev); -static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number); -static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode); -static void nes_terminate_start_timer(struct nes_qp *nesqp); - -static const char *const nes_iwarp_state_str[] = { - "Non-Existent", - "Idle", - "RTS", - "Closing", - "RSVD1", - "Terminate", - "Error", - "RSVD2", -}; - -static const char *const nes_tcp_state_str[] = { - "Non-Existent", - "Closed", - "Listen", - "SYN Sent", - "SYN Rcvd", - "Established", - "Close Wait", - "FIN Wait 1", - "Closing", - "Last Ack", - "FIN Wait 2", - "Time Wait", - "RSVD1", - "RSVD2", - "RSVD3", - "RSVD4", -}; - -static inline void print_ip(struct nes_cm_node *cm_node) -{ - unsigned char *rem_addr; - if (cm_node) { - rem_addr = (unsigned char *)&cm_node->rem_addr; - printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr); - } -} - -/** - * nes_nic_init_timer_defaults - */ -void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW; - shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH; - if (jumbomode) { - shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW; - shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET; - shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH; - } else { - shared_timer->threshold_low = DEFAULT_NES_QL_LOW; - shared_timer->threshold_target = DEFAULT_NES_QL_TARGET; - shared_timer->threshold_high = DEFAULT_NES_QL_HIGH; - } - - /* todo use netdev->mtu to set thresholds */ - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_nic_init_timer - */ -static void nes_nic_init_timer(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - if (shared_timer->timer_in_use_old == 0) { - nesdev->deepcq_count = 0; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - shared_timer->timer_in_use = NES_NIC_FAST_TIMER; - shared_timer->timer_in_use_old = 0; - - } - if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) { - shared_timer->timer_in_use_old = shared_timer->timer_in_use; - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, - 0x80000000 | ((u32)(shared_timer->timer_in_use*8))); - } - /* todo use netdev->mtu to set thresholds */ - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_nic_tune_timer - */ -static void nes_nic_tune_timer(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - u16 cq_count = nesdev->currcq_count; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - if (shared_timer->cq_count_old <= cq_count) - shared_timer->cq_direction_downward = 0; - else - shared_timer->cq_direction_downward++; - shared_timer->cq_count_old = cq_count; - if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) { - if (cq_count <= shared_timer->threshold_low && - shared_timer->threshold_low > 4) { - shared_timer->threshold_low = shared_timer->threshold_low/2; - shared_timer->cq_direction_downward=0; - nesdev->currcq_count = 0; - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - return; - } - } - - if (cq_count > 1) { - nesdev->deepcq_count += cq_count; - if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */ - shared_timer->timer_direction_upward++; - shared_timer->timer_direction_downward = 0; - } else if (cq_count <= shared_timer->threshold_target) { /* balanced */ - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */ - shared_timer->timer_direction_downward++; - shared_timer->timer_direction_upward = 0; - } else if (cq_count <= (shared_timer->threshold_high) * 2) { - shared_timer->timer_in_use -= 2; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward++; - } else { - shared_timer->timer_in_use -= 4; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward++; - } - - if (shared_timer->timer_direction_upward > 3 ) { /* using history */ - shared_timer->timer_in_use += 3; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - } - if (shared_timer->timer_direction_downward > 5) { /* using history */ - shared_timer->timer_in_use -= 4 ; - shared_timer->timer_direction_downward = 0; - shared_timer->timer_direction_upward = 0; - } - } - - /* boundary checking */ - if (shared_timer->timer_in_use > shared_timer->threshold_high) - shared_timer->timer_in_use = shared_timer->threshold_high; - else if (shared_timer->timer_in_use < shared_timer->threshold_low) - shared_timer->timer_in_use = shared_timer->threshold_low; - - nesdev->currcq_count = 0; - - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_init_adapter - initialize adapter - */ -struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) { - struct nes_adapter *nesadapter = NULL; - unsigned long num_pds; - u32 u32temp; - u32 port_count; - u16 max_rq_wrs; - u16 max_sq_wrs; - u32 max_mr; - u32 max_256pbl; - u32 max_4kpbl; - u32 max_qp; - u32 max_irrq; - u32 max_cq; - u32 hte_index_mask; - u32 adapter_size; - u32 arp_table_size; - u16 vendor_id; - u16 device_id; - u8 OneG_Mode; - u8 func_index; - - /* search the list of existing adapters */ - list_for_each_entry(nesadapter, &nes_adapter_list, list) { - nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X," - " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n", - nesdev->pcidev->devfn, - PCI_SLOT(nesadapter->devfn), - nesadapter->bus_number, - PCI_SLOT(nesdev->pcidev->devfn), - nesdev->pcidev->bus->number ); - if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) && - (nesadapter->bus_number == nesdev->pcidev->bus->number)) { - nesadapter->ref_count++; - return nesadapter; - } - } - - /* no adapter found */ - num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT; - if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) { - nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n", - hw_rev); - return NULL; - } - - nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n", - nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8)); - - nes_debug(NES_DBG_INIT, "Reset and init NE020\n"); - - - if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0) - return NULL; - - max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE); - nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp); - - u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE); - if (max_qp > ((u32)1 << (u32temp & 0x001f))) { - nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n", - max_qp, u32temp); - max_qp = (u32)1 << (u32temp & 0x001f); - } - - hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1; - nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n", - max_qp, hte_index_mask); - - u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT); - - max_irrq = 1 << (u32temp & 0x001f); - - if (max_qp > max_irrq) { - max_qp = max_irrq; - nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n", - max_qp); - } - - /* there should be no reason to allocate more pds than qps */ - if (num_pds > max_qp) - num_pds = max_qp; - - u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE); - max_mr = (u32)8192 << (u32temp & 0x7); - - u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE); - max_256pbl = (u32)1 << (u32temp & 0x0000001f); - max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f); - max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE); - - u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE); - arp_table_size = 1 << u32temp; - - adapter_size = (sizeof(struct nes_adapter) + - (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1)); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size); - adapter_size += sizeof(struct nes_qp **) * max_qp; - - /* allocate a new adapter struct */ - nesadapter = kzalloc(adapter_size, GFP_KERNEL); - if (!nesadapter) - return NULL; - - nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n", - nesadapter, (u32)sizeof(struct nes_adapter), adapter_size); - - if (nes_read_eeprom_values(nesdev, nesadapter)) { - printk(KERN_ERR PFX "Unable to read EEPROM data.\n"); - kfree(nesadapter); - return NULL; - } - - nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) | - (nesadapter->mac_addr_low >> 24); - - pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn, - PCI_DEVICE_ID, &device_id); - nesadapter->vendor_part_id = device_id; - - if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter, - OneG_Mode)) { - kfree(nesadapter); - return NULL; - } - nes_init_csr_ne020(nesdev, hw_rev, port_count); - - memset(nesadapter->pft_mcast_map, 255, - sizeof nesadapter->pft_mcast_map); - - /* populate the new nesadapter */ - nesadapter->nesdev = nesdev; - nesadapter->devfn = nesdev->pcidev->devfn; - nesadapter->bus_number = nesdev->pcidev->bus->number; - nesadapter->ref_count = 1; - nesadapter->timer_int_req = 0xffff0000; - nesadapter->OneG_Mode = OneG_Mode; - nesadapter->doorbell_start = nesdev->doorbell_region; - - /* nesadapter->tick_delta = clk_divisor; */ - nesadapter->hw_rev = hw_rev; - nesadapter->port_count = port_count; - - nesadapter->max_qp = max_qp; - nesadapter->hte_index_mask = hte_index_mask; - nesadapter->max_irrq = max_irrq; - nesadapter->max_mr = max_mr; - nesadapter->max_256pbl = max_256pbl - 1; - nesadapter->max_4kpbl = max_4kpbl - 1; - nesadapter->max_cq = max_cq; - nesadapter->free_256pbl = max_256pbl - 1; - nesadapter->free_4kpbl = max_4kpbl - 1; - nesadapter->max_pd = num_pds; - nesadapter->arp_table_size = arp_table_size; - - nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT; - if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) { - nesadapter->et_use_adaptive_rx_coalesce = 0; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT; - nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval; - } else { - nesadapter->et_use_adaptive_rx_coalesce = 1; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC; - nesadapter->et_rx_coalesce_usecs_irq = 0; - printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__); - } - /* Setup and enable the periodic timer */ - if (nesadapter->et_rx_coalesce_usecs_irq) - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 | - ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8))); - else - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000); - - nesadapter->base_pd = 1; - - nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | - IB_DEVICE_MEM_WINDOW | - IB_DEVICE_MEM_MGT_EXTENSIONS; - - nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter) - [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]); - nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)]; - nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)]; - nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)]; - nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)]; - nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]); - - - /* mark the usual suspect QPs, MR and CQs as in use */ - for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) { - set_bit(u32temp, nesadapter->allocated_qps); - set_bit(u32temp, nesadapter->allocated_cqs); - } - set_bit(0, nesadapter->allocated_mrs); - - for (u32temp = 0; u32temp < 20; u32temp++) - set_bit(u32temp, nesadapter->allocated_pds); - u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES); - - max_rq_wrs = ((u32temp >> 8) & 3); - switch (max_rq_wrs) { - case 0: - max_rq_wrs = 4; - break; - case 1: - max_rq_wrs = 16; - break; - case 2: - max_rq_wrs = 32; - break; - case 3: - max_rq_wrs = 512; - break; - } - - max_sq_wrs = (u32temp & 3); - switch (max_sq_wrs) { - case 0: - max_sq_wrs = 4; - break; - case 1: - max_sq_wrs = 16; - break; - case 2: - max_sq_wrs = 32; - break; - case 3: - max_sq_wrs = 512; - break; - } - nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs); - nesadapter->max_irrq_wr = (u32temp >> 16) & 3; - - nesadapter->max_sge = 4; - nesadapter->max_cqe = 32766; - - if (nes_read_eeprom_values(nesdev, nesadapter)) { - printk(KERN_ERR PFX "Unable to read EEPROM data.\n"); - kfree(nesadapter); - return NULL; - } - - u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG); - nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG, - (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff)); - - /* setup port configuration */ - if (nesadapter->port_count == 1) { - nesadapter->log_port = 0x00000000; - if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT) - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002); - else - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003); - } else { - if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) { - nesadapter->log_port = 0x000000D8; - } else { - if (nesadapter->port_count == 2) - nesadapter->log_port = 0x00000044; - else - nesadapter->log_port = 0x000000e4; - } - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003); - } - - nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, - nesadapter->log_port); - nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n", - nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT)); - - spin_lock_init(&nesadapter->resource_lock); - spin_lock_init(&nesadapter->phy_lock); - spin_lock_init(&nesadapter->pbl_lock); - spin_lock_init(&nesadapter->periodic_timer_lock); - - INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]); - - if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) { - u32 pcs_control_status0, pcs_control_status1; - u32 reset_value; - u32 i = 0; - u32 int_cnt = 0; - u32 ext_cnt = 0; - unsigned long flags; - u32 j = 0; - - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - - for (i = 0; i < NES_MAX_LINK_CHECK; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if ((0x0F000100 == (pcs_control_status0 & 0x0F000100)) - || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) - int_cnt++; - usleep_range(1000, 2000); - } - if (int_cnt > 1) { - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - mh_detected++; - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - reset_value |= 0x0000003d; - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (j++ < 5000)); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - - for (i = 0; i < NES_MAX_LINK_CHECK; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if ((0x0F000100 == (pcs_control_status0 & 0x0F000100)) - || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) { - if (++ext_cnt > int_cnt) { - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, - 0x0000F088); - mh_detected++; - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - reset_value |= 0x0000003d; - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (j++ < 5000)); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - break; - } - } - usleep_range(1000, 2000); - } - } - } - - if (nesadapter->hw_rev == NE020_REV) { - timer_setup(&nesadapter->mh_timer, nes_mh_fix, 0); - nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */ - add_timer(&nesadapter->mh_timer); - } else { - nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000); - } - - timer_setup(&nesadapter->lc_timer, nes_clc, 0); - nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */ - add_timer(&nesadapter->lc_timer); - - list_add_tail(&nesadapter->list, &nes_adapter_list); - - for (func_index = 0; func_index < 8; func_index++) { - pci_bus_read_config_word(nesdev->pcidev->bus, - PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn), - func_index), 0, &vendor_id); - if (vendor_id == 0xffff) - break; - } - nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__, - func_index, pci_name(nesdev->pcidev)); - nesadapter->adapter_fcn_count = func_index; - - return nesadapter; -} - - -/** - * nes_reset_adapter_ne020 - */ -static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode) -{ - u32 port_count; - u32 u32temp; - u32 i; - - u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - port_count = ((u32temp & 0x00000300) >> 8) + 1; - /* TODO: assuming that both SERDES are set the same for now */ - *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1; - nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n", - u32temp, port_count); - if (*OneG_Mode) - nes_debug(NES_DBG_INIT, "Running in 1G mode.\n"); - u32temp &= 0xff00ffc0; - switch (port_count) { - case 1: - u32temp |= 0x00ee0000; - break; - case 2: - u32temp |= 0x00cc0000; - break; - case 4: - u32temp |= 0x00000000; - break; - default: - return 0; - break; - } - - /* check and do full reset if needed */ - if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) { - nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd); - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd); - - i = 0; - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000) - mdelay(1); - if (i > 10000) { - nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n"); - return 0; - } - - i = 0; - while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000) - mdelay(1); - if (i > 10000) { - printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n", - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS)); - return 0; - } - } - - /* port reset */ - switch (port_count) { - case 1: - u32temp |= 0x00ee0010; - break; - case 2: - u32temp |= 0x00cc0030; - break; - case 4: - u32temp |= 0x00000030; - break; - } - - nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd); - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd); - - i = 0; - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000) - mdelay(1); - if (i > 10000) { - nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n"); - return 0; - } - - /* serdes 0 */ - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp); - return 0; - } - - /* serdes 1 */ - if (port_count > 1) { - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp); - return 0; - } - } - - return port_count; -} - - -/** - * nes_init_serdes - */ -static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, - struct nes_adapter *nesadapter, u8 OneG_Mode) -{ - int i; - u32 u32temp; - u32 sds; - - if (hw_rev != NE020_REV) { - /* init serdes 0 */ - switch (nesadapter->phy_type[0]) { - case NES_PHY_TYPE_CX4: - if (wide_ppm_offset) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - break; - case NES_PHY_TYPE_KR: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000); - break; - case NES_PHY_TYPE_PUMA_1G: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0); - sds |= 0x00000100; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds); - break; - default: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - break; - } - - if (!OneG_Mode) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000); - - if (port_count < 2) - return 0; - - /* init serdes 1 */ - if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G))) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF); - - switch (nesadapter->phy_type[1]) { - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000); - break; - case NES_PHY_TYPE_CX4: - if (wide_ppm_offset) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA); - break; - case NES_PHY_TYPE_KR: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000); - break; - case NES_PHY_TYPE_PUMA_1G: - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - sds |= 0x000000100; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds); - } - if (!OneG_Mode) { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000); - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - sds &= 0xFFFFFFBF; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds); - } - } else { - /* init serdes 0 */ - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008); - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp); - return 1; - } - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000); - if (OneG_Mode) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222); - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff); - if (port_count > 1) { - /* init serdes 1 */ - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048); - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1) - & 0x0000000f)) != 0x0000000f) && (i++ < 5000)) - mdelay(1); - if (i > 5000) { - printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp); - /* return 1; */ - } - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff); - } - } - return 0; -} - - -/** - * nes_init_csr_ne020 - * Initialize registers for ne020 hardware - */ -static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count) -{ - u32 u32temp; - - nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count); - - nes_write_indexed(nesdev, 0x000001E4, 0x00000007); - /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */ - nes_write_indexed(nesdev, 0x000001E8, 0x00020874); - nes_write_indexed(nesdev, 0x000001D8, 0x00048002); - /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */ - nes_write_indexed(nesdev, 0x000001FC, 0x00050005); - nes_write_indexed(nesdev, 0x00000600, 0x55555555); - nes_write_indexed(nesdev, 0x00000604, 0x55555555); - - /* TODO: move these MAC register settings to NIC bringup */ - nes_write_indexed(nesdev, 0x00002000, 0x00000001); - nes_write_indexed(nesdev, 0x00002004, 0x00000001); - nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000200C, 0x00000001); - nes_write_indexed(nesdev, 0x00002010, 0x000003c1); - nes_write_indexed(nesdev, 0x0000201C, 0x75345678); - if (port_count > 1) { - nes_write_indexed(nesdev, 0x00002200, 0x00000001); - nes_write_indexed(nesdev, 0x00002204, 0x00000001); - nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000220C, 0x00000001); - nes_write_indexed(nesdev, 0x00002210, 0x000003c1); - nes_write_indexed(nesdev, 0x0000221C, 0x75345678); - nes_write_indexed(nesdev, 0x00000908, 0x20000001); - } - if (port_count > 2) { - nes_write_indexed(nesdev, 0x00002400, 0x00000001); - nes_write_indexed(nesdev, 0x00002404, 0x00000001); - nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000240C, 0x00000001); - nes_write_indexed(nesdev, 0x00002410, 0x000003c1); - nes_write_indexed(nesdev, 0x0000241C, 0x75345678); - nes_write_indexed(nesdev, 0x00000910, 0x20000001); - - nes_write_indexed(nesdev, 0x00002600, 0x00000001); - nes_write_indexed(nesdev, 0x00002604, 0x00000001); - nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000260C, 0x00000001); - nes_write_indexed(nesdev, 0x00002610, 0x000003c1); - nes_write_indexed(nesdev, 0x0000261C, 0x75345678); - nes_write_indexed(nesdev, 0x00000918, 0x20000001); - } - - nes_write_indexed(nesdev, 0x00005000, 0x00018000); - /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */ - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) | - 0x00000001); - nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF); - - /* TODO: move this to code, get from EEPROM */ - nes_write_indexed(nesdev, 0x00000900, 0x20000001); - nes_write_indexed(nesdev, 0x000060C0, 0x0000028e); - nes_write_indexed(nesdev, 0x000060C8, 0x00000020); - - nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0); - /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */ - - if (hw_rev != NE020_REV) { - u32temp = nes_read_indexed(nesdev, 0x000008e8); - u32temp |= 0x80000000; - nes_write_indexed(nesdev, 0x000008e8, u32temp); - u32temp = nes_read_indexed(nesdev, 0x000021f8); - u32temp &= 0x7fffffff; - u32temp |= 0x7fff0010; - nes_write_indexed(nesdev, 0x000021f8, u32temp); - if (port_count > 1) { - u32temp = nes_read_indexed(nesdev, 0x000023f8); - u32temp &= 0x7fffffff; - u32temp |= 0x7fff0010; - nes_write_indexed(nesdev, 0x000023f8, u32temp); - } - } -} - - -/** - * nes_destroy_adapter - destroy the adapter structure - */ -void nes_destroy_adapter(struct nes_adapter *nesadapter) -{ - struct nes_adapter *tmp_adapter; - - list_for_each_entry(tmp_adapter, &nes_adapter_list, list) { - nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n", - tmp_adapter); - } - - nesadapter->ref_count--; - if (!nesadapter->ref_count) { - if (nesadapter->hw_rev == NE020_REV) { - del_timer(&nesadapter->mh_timer); - } - del_timer(&nesadapter->lc_timer); - - list_del(&nesadapter->list); - kfree(nesadapter); - } -} - - -/** - * nes_init_cqp - */ -int nes_init_cqp(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_qp_context *cqp_qp_context; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_ceq *ceq; - struct nes_hw_ceq *nic_ceq; - struct nes_hw_aeq *aeq; - void *vmem; - dma_addr_t pmem; - u32 count=0; - u32 cqp_head; - u64 u64temp; - u32 u32temp; - - /* allocate CQP memory */ - /* Need to add max_cq to the aeq size once cq overflow checking is added back */ - /* SQ is 512 byte aligned, others are 256 byte aligned */ - nesdev->cqp_mem_size = 512 + - (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) + - (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) + - max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) + - max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) + - (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) + - sizeof(struct nes_hw_cqp_qp_context); - - nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev, - nesdev->cqp_mem_size, - &nesdev->cqp_pbase); - if (!nesdev->cqp_vbase) { - nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n"); - return -ENOMEM; - } - - /* Allocate a twice the number of CQP requests as the SQ size */ - nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) * - 2 * NES_CQP_SQ_SIZE, GFP_KERNEL); - if (!nesdev->nes_cqp_requests) { - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase, - nesdev->cqp.sq_pbase); - return -ENOMEM; - } - - nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n", - nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size); - - spin_lock_init(&nesdev->cqp.lock); - init_waitqueue_head(&nesdev->cqp.waitq); - - /* Setup Various Structures */ - vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) & - ~(unsigned long)(512 - 1)); - pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) & - ~(unsigned long long)(512 - 1)); - - nesdev->cqp.sq_vbase = vmem; - nesdev->cqp.sq_pbase = pmem; - nesdev->cqp.sq_size = NES_CQP_SQ_SIZE; - nesdev->cqp.sq_head = 0; - nesdev->cqp.sq_tail = 0; - nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn); - - vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size); - pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size); - - nesdev->ccq.cq_vbase = vmem; - nesdev->ccq.cq_pbase = pmem; - nesdev->ccq.cq_size = NES_CCQ_SIZE; - nesdev->ccq.cq_head = 0; - nesdev->ccq.ce_handler = nes_cqp_ce_handler; - nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn); - - vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size); - pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size); - - nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn); - ceq = &nesadapter->ceq[nesdev->ceq_index]; - ceq->ceq_vbase = vmem; - ceq->ceq_pbase = pmem; - ceq->ceq_size = NES_CCEQ_SIZE; - ceq->ceq_head = 0; - - vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256); - pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256); - - nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8; - nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index]; - nic_ceq->ceq_vbase = vmem; - nic_ceq->ceq_pbase = pmem; - nic_ceq->ceq_size = NES_NIC_CEQ_SIZE; - nic_ceq->ceq_head = 0; - - vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256); - pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256); - - aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]; - aeq->aeq_vbase = vmem; - aeq->aeq_pbase = pmem; - aeq->aeq_size = nesadapter->max_qp; - aeq->aeq_head = 0; - - /* Setup QP Context */ - vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size); - pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size); - - cqp_qp_context = vmem; - cqp_qp_context->context_words[0] = - cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10)); - cqp_qp_context->context_words[1] = 0; - cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase); - cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32); - - - /* Write the address to Create CQP */ - if ((sizeof(dma_addr_t) > 4)) { - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), - ((u64)pmem) >> 32); - } else { - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0); - } - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8), - (u32)pmem); - - INIT_LIST_HEAD(&nesdev->cqp_avail_reqs); - INIT_LIST_HEAD(&nesdev->cqp_pending_reqs); - - for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) { - init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq); - list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs); - } - - /* Write Create CCQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nesdev->ccq.cq_number | - ((u32)nesdev->ceq_index << 16))); - u64temp = (u64)nesdev->ccq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&nesdev->ccq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = - cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - - /* Write Create CEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size); - u64temp = (u64)ceq->ceq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Write Create AEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size); - u64temp = (u64)aeq->aeq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Write Create NIC CEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size); - u64temp = (u64)nic_ceq->ceq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Poll until CCQP done */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Error creating CQP\n"); - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, - nesdev->cqp_vbase, nesdev->cqp_pbase); - return -1; - } - udelay(10); - } while (!(nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8))); - - nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - u32temp = 0x04800000; - nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id); - - /* wait for the CCQ, CEQ, and AEQ to get created */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n"); - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, - nesdev->cqp_vbase, nesdev->cqp_pbase); - return -1; - } - udelay(10); - } while (((nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8))); - - /* dump the QP status value */ - nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - nesdev->cqp.sq_tail++; - - return 0; -} - - -/** - * nes_destroy_cqp - */ -int nes_destroy_cqp(struct nes_device *nesdev) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - u32 count = 0; - u32 cqp_head; - unsigned long flags; - - do { - if (count++ > 1000) - break; - udelay(10); - } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail)); - - /* Reset CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET | - nesdev->ccq.cq_number); - - /* Disable device interrupts */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff); - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy the AEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)); - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0; - - /* Destroy the NIC CEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ | - ((u32)nesdev->nic_ceq_index << 8)); - - /* Destroy the CEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ | - (nesdev->ceq_index << 8)); - - /* Destroy the CCQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number | - ((u32)nesdev->ceq_index << 16)); - - /* Destroy CQP */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP | - NES_CQP_QP_TYPE_CQP); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id); - - barrier(); - /* Ring doorbell (5 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - /* wait for the CCQ, CEQ, and AEQ to get destroyed */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n", - PCI_FUNC(nesdev->pcidev->devfn)); - break; - } - udelay(10); - } while (((nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0)); - - /* dump the QP status value */ - nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n", - PCI_FUNC(nesdev->pcidev->devfn), - nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - kfree(nesdev->nes_cqp_requests); - - /* Free the control structures */ - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase, - nesdev->cqp.sq_pbase); - - return 0; -} - - -/** - * nes_init_1g_phy - */ -static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index) -{ - u32 counter = 0; - u16 phy_data; - int ret = 0; - - nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000); - - /* Reset the PHY */ - nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000); - udelay(100); - counter = 0; - do { - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - if (counter++ > 100) { - ret = -1; - break; - } - } while (phy_data & 0x8000); - - /* Setting no phy loopback */ - phy_data &= 0xbfff; - phy_data |= 0x1140; - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data); - nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data); - - /* Setting the interrupt mask */ - nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee); - nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data); - - /* turning on flow control */ - nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00); - nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data); - - /* Clear Half duplex */ - nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100)); - nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data); - - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300); - - return ret; -} - - -/** - * nes_init_2025_phy - */ -static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index) -{ - u32 temp_phy_data = 0; - u32 temp_phy_data2 = 0; - u32 counter = 0; - u32 sds; - u32 mac_index = nesdev->mac_index; - int ret = 0; - unsigned int first_attempt = 1; - - /* Check firmware heartbeat */ - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - udelay(1500); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - if (temp_phy_data != temp_phy_data2) { - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - if ((temp_phy_data & 0xff) > 0x20) - return 0; - printk(PFX "Reinitialize external PHY\n"); - } - - /* no heartbeat, configure the PHY */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - - switch (phy_type) { - case NES_PHY_TYPE_ARGUS: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009); - break; - - case NES_PHY_TYPE_SFP_D: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009); - break; - - case NES_PHY_TYPE_KR: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004); - - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020); - break; - } - - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528); - - /* Bring PHY out of reset */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002); - - /* Check for heartbeat */ - counter = 0; - mdelay(690); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - do { - if (counter++ > 150) { - printk(PFX "No PHY heartbeat\n"); - break; - } - mdelay(1); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - } while (temp_phy_data2 == temp_phy_data); - - /* wait for tracking */ - counter = 0; - do { - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - if (counter++ > 300) { - if (((temp_phy_data & 0xff) == 0x0) && first_attempt) { - first_attempt = 0; - counter = 0; - /* reset AMCC PHY and try again */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040); - continue; - } else { - ret = 1; - break; - } - } - mdelay(10); - } while ((temp_phy_data & 0xff) < 0x30); - - /* setup signal integrity */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032); - if (phy_type == NES_PHY_TYPE_KR) { - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C); - } else { - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063); - } - - /* reset serdes */ - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200); - sds |= 0x1; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds); - sds &= 0xfffffffe; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds); - - counter = 0; - while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040) - && (counter++ < 5000)) - ; - - return ret; -} - - -/** - * nes_init_phy - */ -int nes_init_phy(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 mac_index = nesdev->mac_index; - u32 tx_config = 0; - unsigned long flags; - u8 phy_type = nesadapter->phy_type[mac_index]; - u8 phy_index = nesadapter->phy_index[mac_index]; - int ret = 0; - - tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG); - if (phy_type == NES_PHY_TYPE_1G) { - /* setup 1G MDIO operation */ - tx_config &= 0xFFFFFFE3; - tx_config |= 0x04; - } else { - /* setup 10G MDIO operation */ - tx_config &= 0xFFFFFFE3; - tx_config |= 0x1D; - } - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config); - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - - switch (phy_type) { - case NES_PHY_TYPE_1G: - ret = nes_init_1g_phy(nesdev, phy_type, phy_index); - break; - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - case NES_PHY_TYPE_KR: - ret = nes_init_2025_phy(nesdev, phy_type, phy_index); - break; - } - - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - - return ret; -} - - -/** - * nes_replenish_nic_rq - */ -static void nes_replenish_nic_rq(struct nes_vnic *nesvnic) -{ - unsigned long flags; - dma_addr_t bus_address; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_nic *nesnic; - struct nes_device *nesdev; - struct nes_rskb_cb *cb; - u32 rx_wqes_posted = 0; - - nesnic = &nesvnic->nic; - nesdev = nesvnic->nesdev; - spin_lock_irqsave(&nesnic->rq_lock, flags); - if (nesnic->replenishing_rq !=0) { - if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) && - (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) { - atomic_set(&nesvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */ - add_timer(&nesvnic->rq_wqes_timer); - } else - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - return; - } - nesnic->replenishing_rq = 1; - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - do { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (skb) { - skb->dev = nesvnic->netdev; - - bus_address = pci_map_single(nesdev->pcidev, - skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = bus_address; - cb->maplen = nesvnic->max_frame_size; - - nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = - cpu_to_le32(nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)bus_address); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)bus_address >> 32)); - nesnic->rx_skb[nesnic->rq_head] = skb; - nesnic->rq_head++; - nesnic->rq_head &= nesnic->rq_size - 1; - atomic_dec(&nesvnic->rx_skbs_needed); - barrier(); - if (++rx_wqes_posted == 255) { - nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id); - rx_wqes_posted = 0; - } - } else { - spin_lock_irqsave(&nesnic->rq_lock, flags); - if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) && - (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) { - atomic_set(&nesvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */ - add_timer(&nesvnic->rq_wqes_timer); - } else - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - break; - } - } while (atomic_read(&nesvnic->rx_skbs_needed)); - barrier(); - if (rx_wqes_posted) - nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id); - nesnic->replenishing_rq = 0; -} - - -/** - * nes_rq_wqes_timeout - */ -static void nes_rq_wqes_timeout(struct timer_list *t) -{ - struct nes_vnic *nesvnic = from_timer(nesvnic, t, rq_wqes_timer); - printk("%s: Timer fired.\n", __func__); - atomic_set(&nesvnic->rx_skb_timer_running, 0); - if (atomic_read(&nesvnic->rx_skbs_needed)) - nes_replenish_nic_rq(nesvnic); -} - - -/** - * nes_init_nic_qp - */ -int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct nes_hw_nic_qp_context *nic_context; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_vnic *nesvnic = netdev_priv(netdev); - unsigned long flags; - void *vmem; - dma_addr_t pmem; - u64 u64temp; - int ret; - u32 cqp_head; - u32 counter; - u32 wqe_count; - struct nes_rskb_cb *cb; - u8 jumbomode=0; - - /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */ - nesvnic->nic_mem_size = 256 + - (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) + - (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) + - (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) + - (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) + - sizeof(struct nes_hw_nic_qp_context); - - nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev, - nesvnic->nic_mem_size, - &nesvnic->nic_pbase); - if (!nesvnic->nic_vbase) { - nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n"); - return -ENOMEM; - } - nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n", - nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size); - - vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) & - ~(unsigned long)(256 - 1)); - pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) & - ~(unsigned long long)(256 - 1)); - - /* Setup the first Fragment buffers */ - nesvnic->nic.first_frag_vbase = vmem; - - for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) { - nesvnic->nic.frag_paddr[counter] = pmem; - pmem += sizeof(struct nes_first_frag); - } - - /* setup the SQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)); - - nesvnic->nic.sq_vbase = (void *)vmem; - nesvnic->nic.sq_pbase = pmem; - nesvnic->nic.sq_head = 0; - nesvnic->nic.sq_tail = 0; - nesvnic->nic.sq_size = NES_NIC_WQ_SIZE; - for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) { - nic_sqe = &nesvnic->nic.sq_vbase[counter]; - nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM | - NES_NIC_SQ_WQE_COMPLETION); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] = - cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32)); - } - - nesvnic->get_cqp_request = nes_get_cqp_request; - nesvnic->post_cqp_request = nes_post_cqp_request; - nesvnic->mcrq_mcast_filter = NULL; - - spin_lock_init(&nesvnic->nic.rq_lock); - - /* setup the RQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)); - pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)); - - - nesvnic->nic.rq_vbase = vmem; - nesvnic->nic.rq_pbase = pmem; - nesvnic->nic.rq_head = 0; - nesvnic->nic.rq_tail = 0; - nesvnic->nic.rq_size = NES_NIC_WQ_SIZE; - - /* setup the CQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)); - pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)); - - if (nesdev->nesadapter->netdev_count > 2) - nesvnic->mcrq_qp_id = nesvnic->nic_index + 32; - else - nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4; - - nesvnic->nic_cq.cq_vbase = vmem; - nesvnic->nic_cq.cq_pbase = pmem; - nesvnic->nic_cq.cq_head = 0; - nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2; - - nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler; - - /* Send CreateCQ request to CQP */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - cqp_head = nesdev->cqp.sq_head; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - ((u32)nesvnic->nic_cq.cq_size << 16)); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32( - nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)); - u64temp = (u64)nesvnic->nic_cq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&nesvnic->nic_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* Send CreateQP request to CQP */ - nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]); - nic_context->context_words[NES_NIC_CTX_MISC_IDX] = - cpu_to_le32((u32)NES_NIC_CTX_SIZE | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12)); - nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE), - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE)); - if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) { - nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE); - } - - u64temp = (u64)nesvnic->nic.sq_pbase; - nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)nesvnic->nic.rq_pbase; - nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP | - NES_CQP_QP_TYPE_NIC); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id); - u64temp = (u64)nesvnic->nic_cq.cq_pbase + - (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe)); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - nesdev->cqp.sq_head = cqp_head; - - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n", - nesvnic->nic.qp_id); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n", - nesvnic->nic.qp_id, ret); - if (!ret) { - nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id); - pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase, - nesvnic->nic_pbase); - return -EIO; - } - - /* Populate the RQ */ - for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (!skb) { - nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name); - - nes_destroy_nic_qp(nesvnic); - return -ENOMEM; - } - - skb->dev = netdev; - - pmem = pci_map_single(nesdev->pcidev, skb->data, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = pmem; - cb->maplen = nesvnic->max_frame_size; - - nic_rqe = &nesvnic->nic.rq_vbase[counter]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32)); - nesvnic->nic.rx_skb[counter] = skb; - } - - wqe_count = NES_NIC_WQ_SIZE - 1; - nesvnic->nic.rq_head = wqe_count; - barrier(); - do { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id); - } while (wqe_count); - timer_setup(&nesvnic->rq_wqes_timer, nes_rq_wqes_timeout, 0); - nes_debug(NES_DBG_INIT, "NAPI support Enabled\n"); - if (nesdev->nesadapter->et_use_adaptive_rx_coalesce) - { - nes_nic_init_timer(nesdev); - if (netdev->mtu > 1500) - jumbomode = 1; - nes_nic_init_timer_defaults(nesdev, jumbomode); - } - if ((nesdev->nesadapter->allow_unaligned_fpdus) && - (nes_init_mgt_qp(nesdev, netdev, nesvnic))) { - nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", - netdev->name); - nes_destroy_nic_qp(nesvnic); - return -ENOMEM; - } - - return 0; -} - - -/** - * nes_destroy_nic_qp - */ -void nes_destroy_nic_qp(struct nes_vnic *nesvnic) -{ - u64 u64temp; - dma_addr_t bus_address; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - __le16 *wqe_fragment_length; - u16 wqe_fragment_index; - u32 cqp_head; - u32 wqm_cfg0; - unsigned long flags; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - int ret; - - if (nesdev->nesadapter->allow_unaligned_fpdus) - nes_destroy_mgt(nesvnic); - - /* clear wqe stall before destroying NIC QP */ - wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0); - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF); - - /* Free remaining NIC receive buffers */ - while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) { - rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail]; - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, - PCI_DMA_FROMDEVICE); - - dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]); - nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1); - } - - /* Free remaining NIC transmit buffers */ - while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) { - nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail]; - wqe_fragment_index = 1; - wqe_fragment_length = (__le16 *) - &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* bump past the vlan tag */ - wqe_fragment_length++; - if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) { - u64temp = (u64)le32_to_cpu( - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+ - wqe_fragment_index*2]); - u64temp += ((u64)le32_to_cpu( - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX - + wqe_fragment_index*2]))<<32; - bus_address = (dma_addr_t)u64temp; - if (test_and_clear_bit(nesvnic->nic.sq_tail, - nesvnic->nic.first_frag_overflow)) { - pci_unmap_single(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[ - wqe_fragment_index++]), - PCI_DMA_TODEVICE); - } - for (; wqe_fragment_index < 5; wqe_fragment_index++) { - if (wqe_fragment_length[wqe_fragment_index]) { - u64temp = le32_to_cpu( - nic_sqe->wqe_words[ - NES_NIC_SQ_WQE_FRAG0_LOW_IDX+ - wqe_fragment_index*2]); - u64temp += ((u64)le32_to_cpu( - nic_sqe->wqe_words[ - NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+ - wqe_fragment_index*2]))<<32; - bus_address = (dma_addr_t)u64temp; - pci_unmap_page(nesdev->pcidev, - bus_address, - le16_to_cpu( - wqe_fragment_length[ - wqe_fragment_index]), - PCI_DMA_TODEVICE); - } else - break; - } - } - if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]) - dev_kfree_skb( - nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]); - - nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1) - & (nesvnic->nic.sq_size - 1); - } - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy NIC QP */ - cqp_head = nesdev->cqp.sq_head; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - nesvnic->nic.qp_id); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - - /* Destroy NIC CQ */ - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16))); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - nesdev->cqp.sq_head = cqp_head; - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u," - " cqp.sq_tail=%u, cqp.sq_size=%u\n", - cqp_head, nesdev->cqp.sq_head, - nesdev->cqp.sq_tail, nesdev->cqp.sq_size); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u," - " cqp.sq_head=%u, cqp.sq_tail=%u\n", - ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - if (!ret) { - nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n", - nesvnic->nic.qp_id); - } - - pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase, - nesvnic->nic_pbase); - - /* restore old wqm_cfg0 value */ - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0); -} - -/** - * nes_napi_isr - */ -int nes_napi_isr(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 int_stat; - - if (nesdev->napi_isr_ran) { - /* interrupt status has already been read in ISR */ - int_stat = nesdev->int_stat; - } else { - int_stat = nes_read32(nesdev->regs + NES_INT_STAT); - nesdev->int_stat = int_stat; - nesdev->napi_isr_ran = 1; - } - - int_stat &= nesdev->int_req; - /* iff NIC, process here, else wait for DPC */ - if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) { - nesdev->napi_isr_ran = 0; - nes_write32(nesdev->regs + NES_INT_STAT, - (int_stat & - ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3))); - - /* Process the CEQs */ - nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]); - - if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) && - (!nesadapter->et_use_adaptive_rx_coalesce)) || - ((nesadapter->et_use_adaptive_rx_coalesce) && - (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) { - if ((nesdev->int_req & NES_INT_TIMER) == 0) { - /* Enable Periodic timer interrupts */ - nesdev->int_req |= NES_INT_TIMER; - /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */ - /* TODO: need to also ack other unused periodic timer values, get from nesadapter */ - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INTF_INT_MASK, - ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER)); - } - - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - nes_nic_init_timer(nesdev); - } - /* Enable interrupts, except CEQs */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } else { - /* Enable interrupts, make sure timer is off */ - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - nesdev->deepcq_count = 0; - return 1; - } else { - return 0; - } -} - -static void process_critical_error(struct nes_device *nesdev) -{ - u32 debug_error; - u32 nes_idx_debug_error_masks0 = 0; - u16 error_module = 0; - - debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS); - printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n", - (u16)debug_error); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS, - 0x01010000 | (debug_error & 0x0000ffff)); - if (crit_err_count++ > 10) - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17); - error_module = (u16) (debug_error & 0x1F00) >> 8; - if (++nesdev->nesadapter->crit_error_count[error_module-1] >= - nes_max_critical_error_count) { - printk(KERN_ERR PFX "Masking off critical error for module " - "0x%02X\n", (u16)error_module); - nes_idx_debug_error_masks0 = nes_read_indexed(nesdev, - NES_IDX_DEBUG_ERROR_MASKS0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0, - nes_idx_debug_error_masks0 | (1 << error_module)); - } -} -/** - * nes_dpc - */ -void nes_dpc(unsigned long param) -{ - struct nes_device *nesdev = (struct nes_device *)param; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 counter; - u32 loop_counter = 0; - u32 int_status_bit; - u32 int_stat; - u32 timer_stat; - u32 temp_int_stat; - u32 intf_int_stat; - u32 processed_intf_int = 0; - u16 processed_timer_int = 0; - u16 completion_ints = 0; - u16 timer_ints = 0; - - /* nes_debug(NES_DBG_ISR, "\n"); */ - - do { - timer_stat = 0; - if (nesdev->napi_isr_ran) { - nesdev->napi_isr_ran = 0; - int_stat = nesdev->int_stat; - } else - int_stat = nes_read32(nesdev->regs+NES_INT_STAT); - if (processed_intf_int != 0) - int_stat &= nesdev->int_req & ~NES_INT_INTF; - else - int_stat &= nesdev->int_req; - if (processed_timer_int == 0) { - processed_timer_int = 1; - if (int_stat & NES_INT_TIMER) { - timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT); - if ((timer_stat & nesdev->timer_int_req) == 0) { - int_stat &= ~NES_INT_TIMER; - } - } - } else { - int_stat &= ~NES_INT_TIMER; - } - - if (int_stat) { - if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0| - NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) { - /* Ack the interrupts */ - nes_write32(nesdev->regs+NES_INT_STAT, - (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0| - NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3))); - } - - temp_int_stat = int_stat; - for (counter = 0, int_status_bit = 1; counter < 16; counter++) { - if (int_stat & int_status_bit) { - nes_process_ceq(nesdev, &nesadapter->ceq[counter]); - temp_int_stat &= ~int_status_bit; - completion_ints = 1; - } - if (!(temp_int_stat & 0x0000ffff)) - break; - int_status_bit <<= 1; - } - - /* Process the AEQ for this pci function */ - int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn)); - if (int_stat & int_status_bit) { - nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]); - } - - /* Process the MAC interrupt for this pci function */ - int_status_bit = 1 << (24 + nesdev->mac_index); - if (int_stat & int_status_bit) { - nes_process_mac_intr(nesdev, nesdev->mac_index); - } - - if (int_stat & NES_INT_TIMER) { - if (timer_stat & nesdev->timer_int_req) { - nes_write32(nesdev->regs + NES_TIMER_STAT, - (timer_stat & nesdev->timer_int_req) | - ~(nesdev->nesadapter->timer_int_req)); - timer_ints = 1; - } - } - - if (int_stat & NES_INT_INTF) { - processed_intf_int = 1; - intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT); - intf_int_stat &= nesdev->intf_int_req; - if (NES_INTF_INT_CRITERR & intf_int_stat) { - process_critical_error(nesdev); - } - if (NES_INTF_INT_PCIERR & intf_int_stat) { - printk(KERN_ERR PFX "PCI Error reported by device!!!\n"); - BUG(); - } - if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) { - printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n"); - BUG(); - } - nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat); - } - - if (int_stat & NES_INT_TSW) { - } - } - /* Don't use the interface interrupt bit stay in loop */ - int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | - NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3; - } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS)); - - if (timer_ints == 1) { - if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) { - if (completion_ints == 0) { - nesdev->timer_only_int_count++; - if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) { - nesdev->timer_only_int_count = 0; - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req); - } else { - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } - } else { - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - nes_nic_init_timer(nesdev); - } - nesdev->timer_only_int_count = 0; - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } - } else { - nesdev->timer_only_int_count = 0; - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - } else { - if ( (completion_ints == 1) && - (((nesadapter->et_rx_coalesce_usecs_irq) && - (!nesadapter->et_use_adaptive_rx_coalesce)) || - ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) && - (nesadapter->et_use_adaptive_rx_coalesce) )) ) { - /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */ - nesdev->timer_only_int_count = 0; - nesdev->int_req |= NES_INT_TIMER; - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INTF_INT_MASK, - ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER)); - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } else { - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - } - nesdev->deepcq_count = 0; -} - - -/** - * nes_process_ceq - */ -static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq) -{ - u64 u64temp; - struct nes_hw_cq *cq; - u32 head; - u32 ceq_size; - - /* nes_debug(NES_DBG_CQ, "\n"); */ - head = ceq->ceq_head; - ceq_size = ceq->ceq_size; - - do { - if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) & - NES_CEQE_VALID) { - u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) | - ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX]))); - u64temp <<= 1; - cq = *((struct nes_hw_cq **)&u64temp); - /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */ - barrier(); - ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0; - - /* call the event handler */ - cq->ce_handler(nesdev, cq); - - if (++head >= ceq_size) - head = 0; - } else { - break; - } - - } while (1); - - ceq->ceq_head = head; -} - - -/** - * nes_process_aeq - */ -static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq) -{ - /* u64 u64temp; */ - u32 head; - u32 aeq_size; - u32 aeqe_misc; - u32 aeqe_cq_id; - struct nes_hw_aeqe volatile *aeqe; - - head = aeq->aeq_head; - aeq_size = aeq->aeq_size; - - do { - aeqe = &aeq->aeq_vbase[head]; - if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0) - break; - aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]); - if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) { - if (aeqe_cq_id >= NES_FIRST_QPN) { - /* dealing with an accelerated QP related AE */ - /* - * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) | - * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]))); - */ - nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe); - } else { - /* TODO: dealing with a CQP related AE */ - nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n", - (u16)(aeqe_misc >> 16)); - } - } - - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0; - - if (++head >= aeq_size) - head = 0; - - nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16); - } - while (1); - aeq->aeq_head = head; -} - -static void nes_reset_link(struct nes_device *nesdev, u32 mac_index) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 reset_value; - u32 i=0; - u32 u32temp; - - if (nesadapter->hw_rev == NE020_REV) { - return; - } - mh_detected++; - - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - - if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode))) - reset_value |= 0x0000001d; - else - reset_value |= 0x0000002d; - - if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) { - if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) { - nesadapter->link_interrupt_count[0] = 0; - nesadapter->link_interrupt_count[1] = 0; - u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - if (0x00000040 & u32temp) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - - reset_value |= 0x0000003d; - } - nesadapter->link_interrupt_count[mac_index] = 0; - } - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)); - - if (0x0000003d == (reset_value & 0x0000003d)) { - u32 pcs_control_status0, pcs_control_status1; - - for (i = 0; i < 10; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if (((0x0F000000 == (pcs_control_status0 & 0x0F000000)) - && (pcs_control_status0 & 0x00100000)) - || ((0x0F000000 == (pcs_control_status1 & 0x0F000000)) - && (pcs_control_status1 & 0x00100000))) - continue; - else - break; - } - if (10 == i) { - u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - if (0x00000040 & u32temp) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)); - } - } -} - -/** - * nes_process_mac_intr - */ -static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number) -{ - unsigned long flags; - u32 pcs_control_status; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_vnic *nesvnic; - u32 mac_status; - u32 mac_index = nesdev->mac_index; - u32 u32temp; - u16 phy_data; - u16 temp_phy_data; - u32 pcs_val = 0x0f0f0000; - u32 pcs_mask = 0x0f1f0000; - u32 cdr_ctrl; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) { - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - return; - } - nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT; - - /* ack the MAC interrupt */ - mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200)); - /* Clear the interrupt */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status); - - nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status); - - if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) { - nesdev->link_status_interrupts++; - if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) - nes_reset_link(nesdev, mac_index); - - /* read the PHY interrupt status register */ - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) { - do { - nes_read_1G_phy_reg(nesdev, 0x1a, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - } while (phy_data&0x8000); - - temp_phy_data = 0; - do { - nes_read_1G_phy_reg(nesdev, 0x11, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - if (temp_phy_data == phy_data) - break; - temp_phy_data = phy_data; - } while (1); - - nes_read_1G_phy_reg(nesdev, 0x1e, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - - nes_read_1G_phy_reg(nesdev, 1, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n", - nesadapter->phy_index[mac_index], phy_data); - - if (temp_phy_data & 0x1000) { - nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n"); - phy_data = 4; - } else { - nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n"); - } - } - nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0), - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200)); - - if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) { - switch (mac_index) { - case 1: - case 3: - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - break; - default: - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - break; - } - } else { - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200)); - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200)); - } - - nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n", - mac_index, pcs_control_status); - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) { - u32temp = 0x01010000; - if (nesadapter->port_count > 2) { - u32temp |= 0x02020000; - } - if ((pcs_control_status & u32temp)!= u32temp) { - phy_data = 0; - nes_debug(NES_DBG_PHY, "PCS says the link is down\n"); - } - } else { - switch (nesadapter->phy_type[mac_index]) { - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - case NES_PHY_TYPE_KR: - /* clear the alarms */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005); - /* check link status */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - - nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n", - __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP"); - break; - - case NES_PHY_TYPE_PUMA_1G: - if (mac_index < 2) - pcs_val = pcs_mask = 0x01010000; - else - pcs_val = pcs_mask = 0x02020000; - /* fall through */ - default: - phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0; - break; - } - } - - if (phy_data & 0x0004) { - if (wide_ppm_offset && - (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) && - (nesadapter->hw_rev != NE020_REV)) { - cdr_ctrl = nes_read_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200); - nes_write_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200, - cdr_ctrl | 0x000F0000); - } - nesadapter->mac_link_down[mac_index] = 0; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n", - nesvnic->linkup); - if (nesvnic->linkup == 0) { - printk(PFX "The Link is now up for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (netif_queue_stopped(nesvnic->netdev)) - netif_start_queue(nesvnic->netdev); - nesvnic->linkup = 1; - netif_carrier_on(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } else { - if (wide_ppm_offset && - (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) && - (nesadapter->hw_rev != NE020_REV)) { - cdr_ctrl = nes_read_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200); - nes_write_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200, - cdr_ctrl & 0xFFF0FFFF); - } - nesadapter->mac_link_down[mac_index] = 1; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n", - nesvnic->linkup); - if (nesvnic->linkup == 1) { - printk(PFX "The Link is now down for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (!(netif_queue_stopped(nesvnic->netdev))) - netif_stop_queue(nesvnic->netdev); - nesvnic->linkup = 0; - netif_carrier_off(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 1) { - nesdev->iw_status = 0; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } - if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) { - nesdev->link_recheck = 1; - mod_delayed_work(system_wq, &nesdev->work, - NES_LINK_RECHECK_DELAY); - } - } - - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE; -} - -void nes_recheck_link_status(struct work_struct *work) -{ - unsigned long flags; - struct nes_device *nesdev = container_of(work, struct nes_device, work.work); - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_vnic *nesvnic; - u32 mac_index = nesdev->mac_index; - u16 phy_data; - u16 temp_phy_data; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - - /* check link status */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - - nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n", - __func__, phy_data, - nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP"); - - if (phy_data & 0x0004) { - nesadapter->mac_link_down[mac_index] = 0; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - if (nesvnic->linkup == 0) { - printk(PFX "The Link is now up for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (netif_queue_stopped(nesvnic->netdev)) - netif_start_queue(nesvnic->netdev); - nesvnic->linkup = 1; - netif_carrier_on(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - - } else { - nesadapter->mac_link_down[mac_index] = 1; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - if (nesvnic->linkup == 1) { - printk(PFX "The Link is now down for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (!(netif_queue_stopped(nesvnic->netdev))) - netif_stop_queue(nesvnic->netdev); - nesvnic->linkup = 0; - netif_carrier_off(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 1) { - nesdev->iw_status = 0; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } - if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX) - schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY); - else - nesdev->link_recheck = 0; - - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); -} - - -static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq); - - napi_schedule(&nesvnic->napi); -} - - -/* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before -* getting out of nic_ce_handler -*/ -#define MAX_RQES_TO_PROCESS 384 - -/** - * nes_nic_ce_handler - */ -void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - u64 u64temp; - dma_addr_t bus_address; - struct nes_hw_nic *nesnic; - struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq); - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct sk_buff *skb; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - __le16 *wqe_fragment_length; - u32 head; - u32 cq_size; - u32 rx_pkt_size; - u32 cqe_count=0; - u32 cqe_errv; - u32 cqe_misc; - u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */ - u16 vlan_tag; - u16 pkt_type; - u16 rqes_processed = 0; - u8 sq_cqes = 0; - - head = cq->cq_head; - cq_size = cq->cq_size; - cq->cqes_pending = 1; - do { - if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) & - NES_NIC_CQE_VALID) { - nesnic = &nesvnic->nic; - cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]); - if (cqe_misc & NES_NIC_CQE_SQ) { - sq_cqes++; - wqe_fragment_index = 1; - nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail]; - skb = nesnic->tx_skb[nesnic->sq_tail]; - wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* bump past the vlan tag */ - wqe_fragment_length++; - if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) { - u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX + - wqe_fragment_index * 2]); - u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX + - wqe_fragment_index * 2])) << 32; - bus_address = (dma_addr_t)u64temp; - if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) { - pci_unmap_single(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]), - PCI_DMA_TODEVICE); - } - for (; wqe_fragment_index < 5; wqe_fragment_index++) { - if (wqe_fragment_length[wqe_fragment_index]) { - u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX + - wqe_fragment_index * 2]); - u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX - + wqe_fragment_index * 2])) <<32; - bus_address = (dma_addr_t)u64temp; - pci_unmap_page(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[wqe_fragment_index]), - PCI_DMA_TODEVICE); - } else - break; - } - } - if (skb) - dev_kfree_skb_any(skb); - nesnic->sq_tail++; - nesnic->sq_tail &= nesnic->sq_size-1; - if (sq_cqes > 128) { - barrier(); - /* restart the queue if it had been stopped */ - if (netif_queue_stopped(nesvnic->netdev)) - netif_wake_queue(nesvnic->netdev); - sq_cqes = 0; - } - } else { - rqes_processed ++; - - cq->rx_cqes_completed++; - cq->rx_pkts_indicated++; - rx_pkt_size = cqe_misc & 0x0000ffff; - nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail]; - /* Get the skb */ - rx_skb = nesnic->rx_skb[nesnic->rq_tail]; - nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail]; - bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]); - bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32; - pci_unmap_single(nesdev->pcidev, bus_address, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - cb->busaddr = 0; - /* rx_skb->tail = rx_skb->data + rx_pkt_size; */ - /* rx_skb->len = rx_pkt_size; */ - rx_skb->len = 0; /* TODO: see if this is necessary */ - skb_put(rx_skb, rx_pkt_size); - rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev); - nesnic->rq_tail++; - nesnic->rq_tail &= nesnic->rq_size - 1; - - atomic_inc(&nesvnic->rx_skbs_needed); - if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) { - nes_write32(nesdev->regs+NES_CQE_ALLOC, - cq->cq_number | (cqe_count << 16)); - /* nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - cqe_count = 0; - nes_replenish_nic_rq(nesvnic); - } - pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])); - cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT; - rx_skb->ip_summed = CHECKSUM_NONE; - - if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) || - (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) { - if ((cqe_errv & - (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR | - NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) { - if (nesvnic->netdev->features & NETIF_F_RXCSUM) - rx_skb->ip_summed = CHECKSUM_UNNECESSARY; - } else - nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet." - " errv = 0x%X, pkt_type = 0x%X.\n", - nesvnic->netdev->name, cqe_errv, pkt_type); - - } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) { - if ((cqe_errv & - (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR | - NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) { - if (nesvnic->netdev->features & NETIF_F_RXCSUM) { - rx_skb->ip_summed = CHECKSUM_UNNECESSARY; - /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n", - nesvnic->netdev->name); */ - } - } else - nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet." - " errv = 0x%X, pkt_type = 0x%X.\n", - nesvnic->netdev->name, cqe_errv, pkt_type); - } - /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n", - pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */ - - if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) { - if (nes_cm_recv(rx_skb, nesvnic->netdev)) - rx_skb = NULL; - } - if (rx_skb == NULL) - goto skip_rx_indicate0; - - - if (cqe_misc & NES_NIC_CQE_TAG_VALID) { - vlan_tag = (u16)(le32_to_cpu( - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]) - >> 16); - nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n", - nesvnic->netdev->name, vlan_tag); - - __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag); - } - napi_gro_receive(&nesvnic->napi, rx_skb); - -skip_rx_indicate0: - ; - /* nesvnic->netstats.rx_packets++; */ - /* nesvnic->netstats.rx_bytes += rx_pkt_size; */ - } - - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0; - /* Accounting... */ - cqe_count++; - if (++head >= cq_size) - head = 0; - if (cqe_count == 255) { - /* Replenish Nic CQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, - cq->cq_number | (cqe_count << 16)); - /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - cqe_count = 0; - } - - if (cq->rx_cqes_completed >= nesvnic->budget) - break; - } else { - cq->cqes_pending = 0; - break; - } - - } while (1); - - if (sq_cqes) { - barrier(); - /* restart the queue if it had been stopped */ - if (netif_queue_stopped(nesvnic->netdev)) - netif_wake_queue(nesvnic->netdev); - } - cq->cq_head = head; - /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n", - cq->cq_number, cqe_count, cq->cq_head); */ - cq->cqe_allocs_pending = cqe_count; - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - nes_nic_tune_timer(nesdev); - } - if (atomic_read(&nesvnic->rx_skbs_needed)) - nes_replenish_nic_rq(nesvnic); -} - - - -/** - * nes_cqp_ce_handler - */ -static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq) -{ - u64 u64temp; - unsigned long flags; - struct nes_hw_cqp *cqp = NULL; - struct nes_cqp_request *cqp_request; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 head; - u32 cq_size; - u32 cqe_count=0; - u32 error_code; - u32 opcode; - u32 ctx_index; - /* u32 counter; */ - - head = cq->cq_head; - cq_size = cq->cq_size; - - do { - /* process the CQE */ - /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head, - le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */ - - opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]); - if (opcode & NES_CQE_VALID) { - cqp = &nesdev->cqp; - - error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]); - if (error_code) { - nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP," - " Major/Minor codes = 0x%04X:%04X.\n", - le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f, - (u16)(error_code >> 16), - (u16)error_code); - } - - u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head]. - cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) | - ((u64)(le32_to_cpu(cq->cq_vbase[head]. - cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))); - - cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp; - if (cqp_request) { - if (cqp_request->waiting) { - /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */ - cqp_request->major_code = (u16)(error_code >> 16); - cqp_request->minor_code = (u16)error_code; - barrier(); - cqp_request->request_done = 1; - wake_up(&cqp_request->waitq); - nes_put_cqp_request(nesdev, cqp_request); - } else { - if (cqp_request->callback) - cqp_request->cqp_callback(nesdev, cqp_request); - nes_free_cqp_request(nesdev, cqp_request); - } - } else { - wake_up(&nesdev->cqp.waitq); - } - - cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; - nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16)); - if (++cqp->sq_tail >= cqp->sq_size) - cqp->sq_tail = 0; - - /* Accounting... */ - cqe_count++; - if (++head >= cq_size) - head = 0; - } else { - break; - } - } while (1); - cq->cq_head = head; - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - while ((!list_empty(&nesdev->cqp_pending_reqs)) && - ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) & - (nesdev->cqp.sq_size - 1)) != 1)) { - cqp_request = list_entry(nesdev->cqp_pending_reqs.next, - struct nes_cqp_request, list); - list_del_init(&cqp_request->list); - head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[head]; - memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe)); - barrier(); - - opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]); - if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT) - ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX; - else - ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX; - cqp_wqe->wqe_words[ctx_index] = - cpu_to_le32((u32)((unsigned long)cqp_request)); - cqp_wqe->wqe_words[ctx_index + 1] = - cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request))); - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n", - cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head); - /* Ring doorbell (1 WQEs) */ - barrier(); - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id); - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - /* Arm the CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - cq->cq_number); - nes_read32(nesdev->regs+NES_CQE_ALLOC); -} - -static u8 *locate_mpa(u8 *pkt, u32 aeq_info) -{ - if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) { - /* skip over ethernet header */ - pkt += ETH_HLEN; - - /* Skip over IP and TCP headers */ - pkt += 4 * (pkt[0] & 0x0f); - pkt += 4 * ((pkt[12] >> 4) & 0x0f); - } - return pkt; -} - -/* Determine if incoming error pkt is rdma layer */ -static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info) -{ - u8 *pkt; - u16 *mpa; - u32 opcode = 0xffffffff; - - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - mpa = (u16 *)locate_mpa(pkt, aeq_info); - opcode = be16_to_cpu(mpa[1]) & 0xf; - } - - return opcode; -} - -/* Build iWARP terminate header */ -static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info) -{ - u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - u16 ddp_seg_len; - int copy_len = 0; - u8 is_tagged = 0; - u8 flush_code = 0; - struct nes_terminate_hdr *termhdr; - - termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase; - memset(termhdr, 0, 64); - - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - - /* Use data from offending packet to fill in ddp & rdma hdrs */ - pkt = locate_mpa(pkt, aeq_info); - ddp_seg_len = be16_to_cpu(*(u16 *)pkt); - if (ddp_seg_len) { - copy_len = 2; - termhdr->hdrct = DDP_LEN_FLAG; - if (pkt[2] & 0x80) { - is_tagged = 1; - if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { - copy_len += TERM_DDP_LEN_TAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - } else { - if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { - copy_len += TERM_DDP_LEN_UNTAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - - if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) { - if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) { - copy_len += TERM_RDMA_LEN; - termhdr->hdrct |= RDMA_HDR_FLAG; - } - } - } - } - } - - switch (async_event_id) { - case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_WRITE: - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - } - break; - case NES_AEQE_AEID_AMP_INVALID_STAG: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - break; - case NES_AEQE_AEID_AMP_BAD_QP: - flush_code = IB_WC_LOC_QP_OP_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_QN; - break; - case NES_AEQE_AEID_AMP_BAD_STAG_KEY: - case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_SEND_INV: - case IWARP_OPCODE_SEND_SE_INV: - flush_code = IB_WC_REM_OP_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_CANT_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - } - break; - case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: - if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) { - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_BOUNDS; - } else { - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_BOUNDS; - } - break; - case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: - case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case NES_AEQE_AEID_PRIV_OPERATION_DENIED: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_ACCESS; - break; - case NES_AEQE_AEID_AMP_TO_WRAP: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_TO_WRAP; - break; - case NES_AEQE_AEID_AMP_BAD_PD: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_WRITE: - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_UNASSOC_STAG; - break; - case IWARP_OPCODE_SEND_INV: - case IWARP_OPCODE_SEND_SE_INV: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_CANT_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_UNASSOC_STAG; - } - break; - case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; - termhdr->error_code = MPA_MARKER; - break; - case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; - termhdr->error_code = MPA_CRC; - break; - case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: - case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; - termhdr->error_code = DDP_CATASTROPHIC_LOCAL; - break; - case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: - case NES_AEQE_AEID_DDP_NO_L_BIT: - flush_code = IB_WC_FATAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; - termhdr->error_code = DDP_CATASTROPHIC_LOCAL; - break; - case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: - case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE; - break; - case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: - flush_code = IB_WC_GENERAL_ERR; - if (is_tagged) { - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_INV_DDP_VER; - } else { - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER; - } - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_MO: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MO; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - flush_code = IB_WC_REM_OP_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_QN: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_QN; - break; - case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_INV_RDMAP_VER; - break; - case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: - flush_code = IB_WC_LOC_QP_OP_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_UNEXPECTED_OP; - break; - default: - flush_code = IB_WC_FATAL_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_UNSPECIFIED; - break; - } - - if (copy_len) - memcpy(termhdr + 1, pkt, copy_len); - - if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) { - if (aeq_info & NES_AEQE_SQ) - nesqp->term_sq_flush_code = flush_code; - else - nesqp->term_rq_flush_code = flush_code; - } - - return sizeof(struct nes_terminate_hdr) + copy_len; -} - -static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp, - struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype) -{ - u64 context; - unsigned long flags; - u32 aeq_info; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - u32 termlen = 0; - u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE | - NES_CQP_QP_TERM_DONT_SEND_FIN; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if (nesqp->term_flags & NES_TERM_SENT) - return; /* Sanity check */ - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - async_event_id = (u16)aeq_info; - - context = (unsigned long)nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; - if (!context) { - WARN_ON(!context); - return; - } - - nesqp = (struct nes_qp *)(unsigned long)context; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - nesqp->terminate_eventtype = eventtype; - spin_unlock_irqrestore(&nesqp->lock, flags); - - if (nesadapter->send_term_ok) - termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info); - else - mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG; - - if (!nesdev->iw_status) { - nesqp->term_flags = NES_TERM_DONE; - nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0); - nes_cm_disconn(nesqp); - } else { - nes_terminate_start_timer(nesqp); - nesqp->term_flags |= NES_TERM_SENT; - nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0); - } -} - -static void nes_terminate_send_fin(struct nes_device *nesdev, - struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) -{ - u32 aeq_info; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - unsigned long flags; - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - async_event_id = (u16)aeq_info; - - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - - /* Send the fin only */ - nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE | - NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0); -} - -/* Cleanup after a terminate sent or received */ -static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred) -{ - u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - unsigned long flags; - struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device); - struct nes_device *nesdev = nesvnic->nesdev; - u8 first_time = 0; - - spin_lock_irqsave(&nesqp->lock, flags); - if (nesqp->hte_added) { - nesqp->hte_added = 0; - next_iwarp_state |= NES_CQP_QP_DEL_HTE; - } - - first_time = (nesqp->term_flags & NES_TERM_DONE) == 0; - nesqp->term_flags |= NES_TERM_DONE; - spin_unlock_irqrestore(&nesqp->lock, flags); - - /* Make sure we go through this only once */ - if (first_time) { - if (timeout_occurred == 0) - del_timer(&nesqp->terminate_timer); - else - next_iwarp_state |= NES_CQP_QP_RESET; - - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - } -} - -static void nes_terminate_received(struct nes_device *nesdev, - struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) -{ - u32 aeq_info; - u8 *pkt; - u32 *mpa; - u8 ddp_ctl; - u8 rdma_ctl; - u16 aeq_id = 0; - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - /* Terminate is not a performance path so the silicon */ - /* did not validate the frame - do it now */ - pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - mpa = (u32 *)locate_mpa(pkt, aeq_info); - ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff; - rdma_ctl = be32_to_cpu(mpa[0]) & 0xff; - if ((ddp_ctl & 0xc0) != 0x40) - aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC; - else if ((ddp_ctl & 0x03) != 1) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION; - else if (be32_to_cpu(mpa[2]) != 2) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN; - else if (be32_to_cpu(mpa[3]) != 1) - aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN; - else if (be32_to_cpu(mpa[4]) != 0) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO; - else if ((rdma_ctl & 0xc0) != 0x40) - aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION; - - if (aeq_id) { - /* Bad terminate recvd - send back a terminate */ - aeq_info = (aeq_info & 0xffff0000) | aeq_id; - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); - return; - } - } - - nesqp->term_flags |= NES_TERM_RCVD; - nesqp->terminate_eventtype = IB_EVENT_QP_FATAL; - nes_terminate_start_timer(nesqp); - nes_terminate_send_fin(nesdev, nesqp, aeqe); -} - -/* Timeout routine in case terminate fails to complete */ -void nes_terminate_timeout(struct timer_list *t) -{ - struct nes_qp *nesqp = from_timer(nesqp, t, terminate_timer); - - nes_terminate_done(nesqp, 1); -} - -/* Set a timer in case hw cannot complete the terminate sequence */ -static void nes_terminate_start_timer(struct nes_qp *nesqp) -{ - mod_timer(&nesqp->terminate_timer, (jiffies + HZ)); -} - -/** - * nes_process_iwarp_aeqe - */ -static void nes_process_iwarp_aeqe(struct nes_device *nesdev, - struct nes_hw_aeqe *aeqe) -{ - u64 context; - unsigned long flags; - struct nes_qp *nesqp; - struct nes_hw_cq *hw_cq; - struct nes_cq *nescq; - int resource_allocated; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 aeq_info; - u32 next_iwarp_state = 0; - u32 aeqe_cq_id; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - struct ib_event ibevent; - - nes_debug(NES_DBG_AEQ, "\n"); - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) { - context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]); - context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32; - } else { - context = (unsigned long)nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; - BUG_ON(!context); - } - - /* context is nesqp unless async_event_id == CQ ERROR */ - nesqp = (struct nes_qp *)(unsigned long)context; - async_event_id = (u16)aeq_info; - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p," - " Tcp state = %s, iWARP state = %s\n", - async_event_id, - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe, - nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]); - - aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]); - if (aeq_info & NES_AEQE_QP) { - if (!nes_is_resource_allocated(nesadapter, - nesadapter->allocated_qps, - aeqe_cq_id)) - return; - } - - switch (async_event_id) { - case NES_AEQE_AEID_LLP_FIN_RECEIVED: - if (nesqp->term_flags) - return; /* Ignore it, wait for close complete */ - - if (atomic_inc_return(&nesqp->close_timer_started) == 1) { - if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) && - (nesqp->ibqp_state == IB_QPS_RTS)) { - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - } - nesqp->cm_id->add_ref(nesqp->cm_id); - schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp, - NES_TIMER_TYPE_CLOSE, 1, 0); - nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - async_event_id, nesqp->last_aeq, tcp_state); - } - break; - case NES_AEQE_AEID_LLP_CLOSE_COMPLETE: - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_RESET_SENT: - tcp_state = NES_AEQE_TCP_STATE_CLOSED; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - nesqp->hte_added = 0; - spin_unlock_irqrestore(&nesqp->lock, flags); - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE; - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_LLP_CONNECTION_RESET: - if (atomic_read(&nesqp->close_timer_started)) - return; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_TERMINATE_SENT: - nes_terminate_send_fin(nesdev, nesqp, aeqe); - break; - - case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED: - nes_terminate_received(nesdev, nesqp, aeqe); - break; - - case NES_AEQE_AEID_AMP_BAD_STAG_KEY: - case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: - case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: - case NES_AEQE_AEID_AMP_INVALID_STAG: - case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: - case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case NES_AEQE_AEID_PRIV_OPERATION_DENIED: - case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: - case NES_AEQE_AEID_AMP_TO_WRAP: - printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n", - nesqp->hwqp.qp_id, async_event_id); - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR); - break; - - case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: - case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: - case NES_AEQE_AEID_DDP_UBE_INVALID_MO: - case NES_AEQE_AEID_DDP_UBE_INVALID_QN: - if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) { - aeq_info &= 0xffff0000; - aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE; - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); - } - /* fall through */ - case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE: - case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES: - case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: - case NES_AEQE_AEID_AMP_BAD_QP: - case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: - case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: - case NES_AEQE_AEID_DDP_NO_L_BIT: - case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: - case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: - case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: - case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: - case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: - case NES_AEQE_AEID_AMP_BAD_PD: - case NES_AEQE_AEID_AMP_FASTREG_SHARED: - case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG: - case NES_AEQE_AEID_AMP_FASTREG_MW_STAG: - case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS: - case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW: - case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH: - case NES_AEQE_AEID_AMP_INVALIDATE_SHARED: - case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS: - case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG: - case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG: - case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG: - case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG: - case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS: - case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS: - case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT: - case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED: - case NES_AEQE_AEID_BAD_CLOSE: - case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO: - case NES_AEQE_AEID_STAG_ZERO_INVALID: - case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST: - case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: - printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n", - nesqp->hwqp.qp_id, async_event_id); - print_ip(nesqp->cm_node); - if (!atomic_read(&nesqp->close_timer_started)) - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); - break; - - case NES_AEQE_AEID_CQ_OPERATION_ERROR: - context <<= 1; - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n", - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context); - resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs, - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])); - if (resource_allocated) { - printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n", - __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])); - hw_cq = (struct nes_hw_cq *)(unsigned long)context; - if (hw_cq) { - nescq = container_of(hw_cq, struct nes_cq, hw_cq); - if (nescq->ibcq.event_handler) { - ibevent.device = nescq->ibcq.device; - ibevent.event = IB_EVENT_CQ_ERR; - ibevent.element.cq = &nescq->ibcq; - nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context); - } - } - } - break; - - default: - nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n", - async_event_id); - break; - } - -} - -/** - * nes_iwarp_ce_handler - */ -void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq) -{ - struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq); - - /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n", - nescq->hw_cq.cq_number); */ - nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number); - - if (nescq->ibcq.comp_handler) - nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context); - - return; -} - - -/** - * nes_manage_apbvt() - */ -int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port, - u32 nic_index, u32 add_port) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - int ret = 0; - u16 major_code; - - /* Send manage APBVT request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n", - (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL", - accel_local_port, accel_local_port, nic_index); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT | - ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port)); - - nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n"); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - if (add_port == NES_MANAGE_APBVT_ADD) - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n", - ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - else - return 0; -} - - -/** - * nes_manage_arp_cache - */ -void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr, - u32 ip_addr, u32 action) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev; - struct nes_cqp_request *cqp_request; - int arp_index; - - nesdev = nesvnic->nesdev; - arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action); - if (arp_index == -1) { - return; - } - - /* update the ARP entry */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n"); - return; - } - cqp_request->waiting = 0; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM); - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index); - - if (action == NES_ARP_ADD) { - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID); - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32( - (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) | - (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]); - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32( - (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]); - } else { - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0; - } - - nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n", - nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); -} - - -/** - * flush_wqes - */ -void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp, - u32 which_wq, u32 wait_completion) -{ - struct nes_cqp_request *cqp_request; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; - u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; - int ret; - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return; - } - if (wait_completion) { - cqp_request->waiting = 1; - atomic_set(&cqp_request->refcount, 2); - } else { - cqp_request->waiting = 0; - } - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* If wqe in error was identified, set code to be put into cqe */ - if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) { - which_wq |= NES_CQP_FLUSH_MAJ_MIN; - sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code; - nesqp->term_sq_flush_code = 0; - } - - if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) { - which_wq |= NES_CQP_FLUSH_MAJ_MIN; - rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code; - nesqp->term_rq_flush_code = 0; - } - - if (which_wq & NES_CQP_FLUSH_MAJ_MIN) { - cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code); - cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code); - } - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = - cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id); - - nes_post_cqp_request(nesdev, cqp_request); - - if (wait_completion) { - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u," - " CQP Major:Minor codes = 0x%04X:0x%04X\n", - ret, cqp_request->major_code, cqp_request->minor_code); - nes_put_cqp_request(nesdev, cqp_request); - } -} diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h deleted file mode 100644 index 3c56470816a8..000000000000 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ /dev/null @@ -1,1380 +0,0 @@ -/* -* Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenIB.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef __NES_HW_H -#define __NES_HW_H - -#define NES_PHY_TYPE_CX4 1 -#define NES_PHY_TYPE_1G 2 -#define NES_PHY_TYPE_ARGUS 4 -#define NES_PHY_TYPE_PUMA_1G 5 -#define NES_PHY_TYPE_PUMA_10G 6 -#define NES_PHY_TYPE_GLADIUS 7 -#define NES_PHY_TYPE_SFP_D 8 -#define NES_PHY_TYPE_KR 9 - -#define NES_MULTICAST_PF_MAX 8 -#define NES_A0 3 - -#define NES_ENABLE_PAU 0x07000001 -#define NES_DISABLE_PAU 0x07000000 -#define NES_PAU_COUNTER 10 -#define NES_CQP_OPCODE_MASK 0x3f - -enum pci_regs { - NES_INT_STAT = 0x0000, - NES_INT_MASK = 0x0004, - NES_INT_PENDING = 0x0008, - NES_INTF_INT_STAT = 0x000C, - NES_INTF_INT_MASK = 0x0010, - NES_TIMER_STAT = 0x0014, - NES_PERIODIC_CONTROL = 0x0018, - NES_ONE_SHOT_CONTROL = 0x001C, - NES_EEPROM_COMMAND = 0x0020, - NES_EEPROM_DATA = 0x0024, - NES_FLASH_COMMAND = 0x0028, - NES_FLASH_DATA = 0x002C, - NES_SOFTWARE_RESET = 0x0030, - NES_CQ_ACK = 0x0034, - NES_WQE_ALLOC = 0x0040, - NES_CQE_ALLOC = 0x0044, - NES_AEQ_ALLOC = 0x0048 -}; - -enum indexed_regs { - NES_IDX_CREATE_CQP_LOW = 0x0000, - NES_IDX_CREATE_CQP_HIGH = 0x0004, - NES_IDX_QP_CONTROL = 0x0040, - NES_IDX_FLM_CONTROL = 0x0080, - NES_IDX_INT_CPU_STATUS = 0x00a0, - NES_IDX_GPR_TRIGGER = 0x00bc, - NES_IDX_GPIO_CONTROL = 0x00f0, - NES_IDX_GPIO_DATA = 0x00f4, - NES_IDX_GPR2 = 0x010c, - NES_IDX_TCP_CONFIG0 = 0x01e4, - NES_IDX_TCP_TIMER_CONFIG = 0x01ec, - NES_IDX_TCP_NOW = 0x01f0, - NES_IDX_QP_MAX_CFG_SIZES = 0x0200, - NES_IDX_QP_CTX_SIZE = 0x0218, - NES_IDX_TCP_TIMER_SIZE0 = 0x0238, - NES_IDX_TCP_TIMER_SIZE1 = 0x0240, - NES_IDX_ARP_CACHE_SIZE = 0x0258, - NES_IDX_CQ_CTX_SIZE = 0x0260, - NES_IDX_MRT_SIZE = 0x0278, - NES_IDX_PBL_REGION_SIZE = 0x0280, - NES_IDX_IRRQ_COUNT = 0x02b0, - NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x02f0, - NES_IDX_RX_WINDOW_BUFFER_SIZE = 0x0300, - NES_IDX_DST_IP_ADDR = 0x0400, - NES_IDX_PCIX_DIAG = 0x08e8, - NES_IDX_MPP_DEBUG = 0x0a00, - NES_IDX_PORT_RX_DISCARDS = 0x0a30, - NES_IDX_PORT_TX_DISCARDS = 0x0a34, - NES_IDX_MPP_LB_DEBUG = 0x0b00, - NES_IDX_DENALI_CTL_22 = 0x1058, - NES_IDX_MAC_TX_CONTROL = 0x2000, - NES_IDX_MAC_TX_CONFIG = 0x2004, - NES_IDX_MAC_TX_PAUSE_QUANTA = 0x2008, - NES_IDX_MAC_RX_CONTROL = 0x200c, - NES_IDX_MAC_RX_CONFIG = 0x2010, - NES_IDX_MAC_EXACT_MATCH_BOTTOM = 0x201c, - NES_IDX_MAC_MDIO_CONTROL = 0x2084, - NES_IDX_MAC_TX_OCTETS_LOW = 0x2100, - NES_IDX_MAC_TX_OCTETS_HIGH = 0x2104, - NES_IDX_MAC_TX_FRAMES_LOW = 0x2108, - NES_IDX_MAC_TX_FRAMES_HIGH = 0x210c, - NES_IDX_MAC_TX_PAUSE_FRAMES = 0x2118, - NES_IDX_MAC_TX_ERRORS = 0x2138, - NES_IDX_MAC_RX_OCTETS_LOW = 0x213c, - NES_IDX_MAC_RX_OCTETS_HIGH = 0x2140, - NES_IDX_MAC_RX_FRAMES_LOW = 0x2144, - NES_IDX_MAC_RX_FRAMES_HIGH = 0x2148, - NES_IDX_MAC_RX_BC_FRAMES_LOW = 0x214c, - NES_IDX_MAC_RX_MC_FRAMES_HIGH = 0x2150, - NES_IDX_MAC_RX_PAUSE_FRAMES = 0x2154, - NES_IDX_MAC_RX_SHORT_FRAMES = 0x2174, - NES_IDX_MAC_RX_OVERSIZED_FRAMES = 0x2178, - NES_IDX_MAC_RX_JABBER_FRAMES = 0x217c, - NES_IDX_MAC_RX_CRC_ERR_FRAMES = 0x2180, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES = 0x2184, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES = 0x2188, - NES_IDX_MAC_INT_STATUS = 0x21f0, - NES_IDX_MAC_INT_MASK = 0x21f4, - NES_IDX_PHY_PCS_CONTROL_STATUS0 = 0x2800, - NES_IDX_PHY_PCS_CONTROL_STATUS1 = 0x2a00, - NES_IDX_ETH_SERDES_COMMON_CONTROL0 = 0x2808, - NES_IDX_ETH_SERDES_COMMON_CONTROL1 = 0x2a08, - NES_IDX_ETH_SERDES_COMMON_STATUS0 = 0x280c, - NES_IDX_ETH_SERDES_COMMON_STATUS1 = 0x2a0c, - NES_IDX_ETH_SERDES_TX_EMP0 = 0x2810, - NES_IDX_ETH_SERDES_TX_EMP1 = 0x2a10, - NES_IDX_ETH_SERDES_TX_DRIVE0 = 0x2814, - NES_IDX_ETH_SERDES_TX_DRIVE1 = 0x2a14, - NES_IDX_ETH_SERDES_RX_MODE0 = 0x2818, - NES_IDX_ETH_SERDES_RX_MODE1 = 0x2a18, - NES_IDX_ETH_SERDES_RX_SIGDET0 = 0x281c, - NES_IDX_ETH_SERDES_RX_SIGDET1 = 0x2a1c, - NES_IDX_ETH_SERDES_BYPASS0 = 0x2820, - NES_IDX_ETH_SERDES_BYPASS1 = 0x2a20, - NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0 = 0x2824, - NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1 = 0x2a24, - NES_IDX_ETH_SERDES_RX_EQ_CONTROL0 = 0x2828, - NES_IDX_ETH_SERDES_RX_EQ_CONTROL1 = 0x2a28, - NES_IDX_ETH_SERDES_RX_EQ_STATUS0 = 0x282c, - NES_IDX_ETH_SERDES_RX_EQ_STATUS1 = 0x2a2c, - NES_IDX_ETH_SERDES_CDR_RESET0 = 0x2830, - NES_IDX_ETH_SERDES_CDR_RESET1 = 0x2a30, - NES_IDX_ETH_SERDES_CDR_CONTROL0 = 0x2834, - NES_IDX_ETH_SERDES_CDR_CONTROL1 = 0x2a34, - NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0 = 0x2838, - NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1 = 0x2a38, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD = 0x3080, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO = 0x3000, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI = 0x3004, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO = 0x3008, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI = 0x300c, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO = 0x7000, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI = 0x7004, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO = 0x7008, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI = 0x700c, - NES_IDX_WQM_CONFIG0 = 0x5000, - NES_IDX_WQM_CONFIG1 = 0x5004, - NES_IDX_CM_CONFIG = 0x5100, - NES_IDX_NIC_LOGPORT_TO_PHYPORT = 0x6000, - NES_IDX_NIC_PHYPORT_TO_USW = 0x6008, - NES_IDX_NIC_ACTIVE = 0x6010, - NES_IDX_NIC_UNICAST_ALL = 0x6018, - NES_IDX_NIC_MULTICAST_ALL = 0x6020, - NES_IDX_NIC_MULTICAST_ENABLE = 0x6028, - NES_IDX_NIC_BROADCAST_ON = 0x6030, - NES_IDX_USED_CHUNKS_TX = 0x60b0, - NES_IDX_TX_POOL_SIZE = 0x60b8, - NES_IDX_QUAD_HASH_TABLE_SIZE = 0x6148, - NES_IDX_PERFECT_FILTER_LOW = 0x6200, - NES_IDX_PERFECT_FILTER_HIGH = 0x6204, - NES_IDX_IPV4_TCP_REXMITS = 0x7080, - NES_IDX_DEBUG_ERROR_CONTROL_STATUS = 0x913c, - NES_IDX_DEBUG_ERROR_MASKS0 = 0x9140, - NES_IDX_DEBUG_ERROR_MASKS1 = 0x9144, - NES_IDX_DEBUG_ERROR_MASKS2 = 0x9148, - NES_IDX_DEBUG_ERROR_MASKS3 = 0x914c, - NES_IDX_DEBUG_ERROR_MASKS4 = 0x9150, - NES_IDX_DEBUG_ERROR_MASKS5 = 0x9154, -}; - -#define NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE 1 -#define NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE (1 << 17) - -enum nes_cqp_opcodes { - NES_CQP_CREATE_QP = 0x00, - NES_CQP_MODIFY_QP = 0x01, - NES_CQP_DESTROY_QP = 0x02, - NES_CQP_CREATE_CQ = 0x03, - NES_CQP_MODIFY_CQ = 0x04, - NES_CQP_DESTROY_CQ = 0x05, - NES_CQP_ALLOCATE_STAG = 0x09, - NES_CQP_REGISTER_STAG = 0x0a, - NES_CQP_QUERY_STAG = 0x0b, - NES_CQP_REGISTER_SHARED_STAG = 0x0c, - NES_CQP_DEALLOCATE_STAG = 0x0d, - NES_CQP_MANAGE_ARP_CACHE = 0x0f, - NES_CQP_DOWNLOAD_SEGMENT = 0x10, - NES_CQP_SUSPEND_QPS = 0x11, - NES_CQP_UPLOAD_CONTEXT = 0x13, - NES_CQP_CREATE_CEQ = 0x16, - NES_CQP_DESTROY_CEQ = 0x18, - NES_CQP_CREATE_AEQ = 0x19, - NES_CQP_DESTROY_AEQ = 0x1b, - NES_CQP_LMI_ACCESS = 0x20, - NES_CQP_FLUSH_WQES = 0x22, - NES_CQP_MANAGE_APBVT = 0x23, - NES_CQP_MANAGE_QUAD_HASH = 0x25 -}; - -enum nes_cqp_wqe_word_idx { - NES_CQP_WQE_OPCODE_IDX = 0, - NES_CQP_WQE_ID_IDX = 1, - NES_CQP_WQE_COMP_CTX_LOW_IDX = 2, - NES_CQP_WQE_COMP_CTX_HIGH_IDX = 3, - NES_CQP_WQE_COMP_SCRATCH_LOW_IDX = 4, - NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX = 5, -}; - -enum nes_cqp_wqe_word_download_idx { /* format differs from other cqp ops */ - NES_CQP_WQE_DL_OPCODE_IDX = 0, - NES_CQP_WQE_DL_COMP_CTX_LOW_IDX = 1, - NES_CQP_WQE_DL_COMP_CTX_HIGH_IDX = 2, - NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX = 3 - /* For index values 4-15 use NES_NIC_SQ_WQE_ values */ -}; - -enum nes_cqp_cq_wqeword_idx { - NES_CQP_CQ_WQE_PBL_LOW_IDX = 6, - NES_CQP_CQ_WQE_PBL_HIGH_IDX = 7, - NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX = 8, - NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX = 9, - NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX = 10, -}; - -enum nes_cqp_stag_wqeword_idx { - NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX = 1, - NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX = 6, - NES_CQP_STAG_WQE_LEN_LOW_IDX = 7, - NES_CQP_STAG_WQE_STAG_IDX = 8, - NES_CQP_STAG_WQE_VA_LOW_IDX = 10, - NES_CQP_STAG_WQE_VA_HIGH_IDX = 11, - NES_CQP_STAG_WQE_PA_LOW_IDX = 12, - NES_CQP_STAG_WQE_PA_HIGH_IDX = 13, - NES_CQP_STAG_WQE_PBL_LEN_IDX = 14 -}; - -#define NES_CQP_OP_LOGICAL_PORT_SHIFT 26 -#define NES_CQP_OP_IWARP_STATE_SHIFT 28 -#define NES_CQP_OP_TERMLEN_SHIFT 28 - -enum nes_cqp_qp_bits { - NES_CQP_QP_ARP_VALID = (1<<8), - NES_CQP_QP_WINBUF_VALID = (1<<9), - NES_CQP_QP_CONTEXT_VALID = (1<<10), - NES_CQP_QP_ORD_VALID = (1<<11), - NES_CQP_QP_WINBUF_DATAIND_EN = (1<<12), - NES_CQP_QP_VIRT_WQS = (1<<13), - NES_CQP_QP_DEL_HTE = (1<<14), - NES_CQP_QP_CQS_VALID = (1<<15), - NES_CQP_QP_TYPE_TSA = 0, - NES_CQP_QP_TYPE_IWARP = (1<<16), - NES_CQP_QP_TYPE_CQP = (4<<16), - NES_CQP_QP_TYPE_NIC = (5<<16), - NES_CQP_QP_MSS_CHG = (1<<20), - NES_CQP_QP_STATIC_RESOURCES = (1<<21), - NES_CQP_QP_IGNORE_MW_BOUND = (1<<22), - NES_CQP_QP_VWQ_USE_LMI = (1<<23), - NES_CQP_QP_IWARP_STATE_IDLE = (1<netdev */ - u8 perfect_filter_index; - u8 nic_index; - u8 qp_nic_index[4]; - u8 next_qp_nic_index; - u8 of_device_registered; - u8 rdma_enabled; - struct timer_list event_timer; - enum ib_event_type delayed_event; - enum ib_event_type last_dispatched_event; - spinlock_t port_ibevent_lock; - u32 mgt_mem_size; - void *mgt_vbase; - dma_addr_t mgt_pbase; - struct nes_vnic_mgt *mgtvnic[NES_MGT_QP_COUNT]; - struct task_struct *mgt_thread; - wait_queue_head_t mgt_wait_queue; - struct sk_buff_head mgt_skb_list; - -}; - -struct nes_ib_device { - struct ib_device ibdev; - struct nes_vnic *nesvnic; - - /* Virtual RNIC Limits */ - u32 max_mr; - u32 max_qp; - u32 max_cq; - u32 max_pd; - u32 num_mr; - u32 num_qp; - u32 num_cq; - u32 num_pd; -}; - -enum nes_hdrct_flags { - DDP_LEN_FLAG = 0x80, - DDP_HDR_FLAG = 0x40, - RDMA_HDR_FLAG = 0x20 -}; - -enum nes_term_layers { - LAYER_RDMA = 0, - LAYER_DDP = 1, - LAYER_MPA = 2 -}; - -enum nes_term_error_types { - RDMAP_CATASTROPHIC = 0, - RDMAP_REMOTE_PROT = 1, - RDMAP_REMOTE_OP = 2, - DDP_CATASTROPHIC = 0, - DDP_TAGGED_BUFFER = 1, - DDP_UNTAGGED_BUFFER = 2, - DDP_LLP = 3 -}; - -enum nes_term_rdma_errors { - RDMAP_INV_STAG = 0x00, - RDMAP_INV_BOUNDS = 0x01, - RDMAP_ACCESS = 0x02, - RDMAP_UNASSOC_STAG = 0x03, - RDMAP_TO_WRAP = 0x04, - RDMAP_INV_RDMAP_VER = 0x05, - RDMAP_UNEXPECTED_OP = 0x06, - RDMAP_CATASTROPHIC_LOCAL = 0x07, - RDMAP_CATASTROPHIC_GLOBAL = 0x08, - RDMAP_CANT_INV_STAG = 0x09, - RDMAP_UNSPECIFIED = 0xff -}; - -enum nes_term_ddp_errors { - DDP_CATASTROPHIC_LOCAL = 0x00, - DDP_TAGGED_INV_STAG = 0x00, - DDP_TAGGED_BOUNDS = 0x01, - DDP_TAGGED_UNASSOC_STAG = 0x02, - DDP_TAGGED_TO_WRAP = 0x03, - DDP_TAGGED_INV_DDP_VER = 0x04, - DDP_UNTAGGED_INV_QN = 0x01, - DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02, - DDP_UNTAGGED_INV_MSN_RANGE = 0x03, - DDP_UNTAGGED_INV_MO = 0x04, - DDP_UNTAGGED_INV_TOO_LONG = 0x05, - DDP_UNTAGGED_INV_DDP_VER = 0x06 -}; - -enum nes_term_mpa_errors { - MPA_CLOSED = 0x01, - MPA_CRC = 0x02, - MPA_MARKER = 0x03, - MPA_REQ_RSP = 0x04, -}; - -struct nes_terminate_hdr { - u8 layer_etype; - u8 error_code; - u8 hdrct; - u8 rsvd; -}; - -/* Used to determine how to fill in terminate error codes */ -#define IWARP_OPCODE_WRITE 0 -#define IWARP_OPCODE_READREQ 1 -#define IWARP_OPCODE_READRSP 2 -#define IWARP_OPCODE_SEND 3 -#define IWARP_OPCODE_SEND_INV 4 -#define IWARP_OPCODE_SEND_SE 5 -#define IWARP_OPCODE_SEND_SE_INV 6 -#define IWARP_OPCODE_TERM 7 - -/* These values are used only during terminate processing */ -#define TERM_DDP_LEN_TAGGED 14 -#define TERM_DDP_LEN_UNTAGGED 18 -#define TERM_RDMA_LEN 28 -#define RDMA_OPCODE_MASK 0x0f -#define RDMA_READ_REQ_OPCODE 1 -#define BAD_FRAME_OFFSET 64 -#define CQE_MAJOR_DRV 0x8000 - -/* Used for link status recheck after interrupt processing */ -#define NES_LINK_RECHECK_DELAY msecs_to_jiffies(50) -#define NES_LINK_RECHECK_MAX 60 - -#endif /* __NES_HW_H */ diff --git a/drivers/infiniband/hw/nes/nes_mgt.c b/drivers/infiniband/hw/nes/nes_mgt.c deleted file mode 100644 index cc4dce5c3e5f..000000000000 --- a/drivers/infiniband/hw/nes/nes_mgt.c +++ /dev/null @@ -1,1155 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include -#include "nes.h" -#include "nes_mgt.h" - -atomic_t pau_qps_created; -atomic_t pau_qps_destroyed; - -static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic) -{ - unsigned long flags; - dma_addr_t bus_address; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_mgt *nesmgt; - struct nes_device *nesdev; - struct nes_rskb_cb *cb; - u32 rx_wqes_posted = 0; - - nesmgt = &mgtvnic->mgt; - nesdev = mgtvnic->nesvnic->nesdev; - spin_lock_irqsave(&nesmgt->rq_lock, flags); - if (nesmgt->replenishing_rq != 0) { - if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) && - (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) { - atomic_set(&mgtvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */ - add_timer(&mgtvnic->rq_wqes_timer); - } else { - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - } - return; - } - nesmgt->replenishing_rq = 1; - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - do { - skb = dev_alloc_skb(mgtvnic->nesvnic->max_frame_size); - if (skb) { - skb->dev = mgtvnic->nesvnic->netdev; - - bus_address = pci_map_single(nesdev->pcidev, - skb->data, mgtvnic->nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = bus_address; - cb->maplen = mgtvnic->nesvnic->max_frame_size; - - nic_rqe = &nesmgt->rq_vbase[mgtvnic->mgt.rq_head]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = - cpu_to_le32(mgtvnic->nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)bus_address); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)bus_address >> 32)); - nesmgt->rx_skb[nesmgt->rq_head] = skb; - nesmgt->rq_head++; - nesmgt->rq_head &= nesmgt->rq_size - 1; - atomic_dec(&mgtvnic->rx_skbs_needed); - barrier(); - if (++rx_wqes_posted == 255) { - nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id); - rx_wqes_posted = 0; - } - } else { - spin_lock_irqsave(&nesmgt->rq_lock, flags); - if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) && - (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) { - atomic_set(&mgtvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */ - add_timer(&mgtvnic->rq_wqes_timer); - } else { - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - } - break; - } - } while (atomic_read(&mgtvnic->rx_skbs_needed)); - barrier(); - if (rx_wqes_posted) - nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id); - nesmgt->replenishing_rq = 0; -} - -/** - * nes_mgt_rq_wqes_timeout - */ -static void nes_mgt_rq_wqes_timeout(struct timer_list *t) -{ - struct nes_vnic_mgt *mgtvnic = from_timer(mgtvnic, t, - rq_wqes_timer); - - atomic_set(&mgtvnic->rx_skb_timer_running, 0); - if (atomic_read(&mgtvnic->rx_skbs_needed)) - nes_replenish_mgt_rq(mgtvnic); -} - -/** - * nes_mgt_free_skb - unmap and free skb - */ -static void nes_mgt_free_skb(struct nes_device *nesdev, struct sk_buff *skb, u32 dir) -{ - struct nes_rskb_cb *cb; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, dir); - cb->busaddr = 0; - dev_kfree_skb_any(skb); -} - -/** - * nes_download_callback - handle download completions - */ -static void nes_download_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - struct pau_fpdu_info *fpdu_info = cqp_request->cqp_callback_pointer; - struct nes_qp *nesqp = fpdu_info->nesqp; - struct sk_buff *skb; - int i; - - for (i = 0; i < fpdu_info->frag_cnt; i++) { - skb = fpdu_info->frags[i].skb; - if (fpdu_info->frags[i].cmplt) { - nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - } - } - - if (fpdu_info->hdr_vbase) - pci_free_consistent(nesdev->pcidev, fpdu_info->hdr_len, - fpdu_info->hdr_vbase, fpdu_info->hdr_pbase); - kfree(fpdu_info); -} - -/** - * nes_get_seq - Get the seq, ack_seq and window from the packet - */ -static u32 nes_get_seq(struct sk_buff *skb, u32 *ack, u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd) -{ - struct nes_rskb_cb *cb = (struct nes_rskb_cb *)&skb->cb[0]; - struct iphdr *iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - struct tcphdr *tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - - *ack = be32_to_cpu(tcph->ack_seq); - *wnd = be16_to_cpu(tcph->window); - *fin_rcvd = tcph->fin; - *rst_rcvd = tcph->rst; - return be32_to_cpu(tcph->seq); -} - -/** - * nes_get_next_skb - Get the next skb based on where current skb is in the queue - */ -static struct sk_buff *nes_get_next_skb(struct nes_device *nesdev, struct nes_qp *nesqp, - struct sk_buff *skb, u32 nextseq, u32 *ack, - u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd) -{ - u32 seq; - bool processacks; - struct sk_buff *old_skb; - - if (skb) { - /* Continue processing fpdu */ - skb = skb_peek_next(skb, &nesqp->pau_list); - if (!skb) - goto out; - processacks = false; - } else { - /* Starting a new one */ - if (skb_queue_empty(&nesqp->pau_list)) - goto out; - skb = skb_peek(&nesqp->pau_list); - processacks = true; - } - - while (1) { - if (skb_queue_empty(&nesqp->pau_list)) - goto out; - - seq = nes_get_seq(skb, ack, wnd, fin_rcvd, rst_rcvd); - if (seq == nextseq) { - if (skb->len || processacks) - break; - } else if (after(seq, nextseq)) { - goto out; - } - - old_skb = skb; - skb = skb_peek_next(skb, &nesqp->pau_list); - skb_unlink(old_skb, &nesqp->pau_list); - nes_mgt_free_skb(nesdev, old_skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - if (!skb) - goto out; - } - return skb; - -out: - return NULL; -} - -/** - * get_fpdu_info - Find the next complete fpdu and return its fragments. - */ -static int get_fpdu_info(struct nes_device *nesdev, struct nes_qp *nesqp, - struct pau_fpdu_info **pau_fpdu_info) -{ - struct sk_buff *skb; - struct iphdr *iph; - struct tcphdr *tcph; - struct nes_rskb_cb *cb; - struct pau_fpdu_info *fpdu_info = NULL; - struct pau_fpdu_frag frags[MAX_FPDU_FRAGS]; - u32 fpdu_len = 0; - u32 tmp_len; - int frag_cnt = 0; - u32 tot_len; - u32 frag_tot; - u32 ack; - u32 fin_rcvd; - u32 rst_rcvd; - u16 wnd; - int i; - int rc = 0; - - *pau_fpdu_info = NULL; - - skb = nes_get_next_skb(nesdev, nesqp, NULL, nesqp->pau_rcv_nxt, &ack, &wnd, &fin_rcvd, &rst_rcvd); - if (!skb) - goto out; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - if (skb->len) { - fpdu_len = be16_to_cpu(*(__be16 *) skb->data) + MPA_FRAMING; - fpdu_len = (fpdu_len + 3) & 0xfffffffc; - tmp_len = fpdu_len; - - /* See if we have all of the fpdu */ - frag_tot = 0; - memset(&frags, 0, sizeof frags); - for (i = 0; i < MAX_FPDU_FRAGS; i++) { - frags[i].physaddr = cb->busaddr; - frags[i].physaddr += skb->data - cb->data_start; - frags[i].frag_len = min(tmp_len, skb->len); - frags[i].skb = skb; - frags[i].cmplt = (skb->len == frags[i].frag_len); - frag_tot += frags[i].frag_len; - frag_cnt++; - - tmp_len -= frags[i].frag_len; - if (tmp_len == 0) - break; - - skb = nes_get_next_skb(nesdev, nesqp, skb, - nesqp->pau_rcv_nxt + frag_tot, &ack, &wnd, &fin_rcvd, &rst_rcvd); - if (!skb) - goto out; - if (rst_rcvd) { - /* rst received in the middle of fpdu */ - for (; i >= 0; i--) { - skb_unlink(frags[i].skb, &nesqp->pau_list); - nes_mgt_free_skb(nesdev, frags[i].skb, PCI_DMA_TODEVICE); - } - cb = (struct nes_rskb_cb *)&skb->cb[0]; - frags[0].physaddr = cb->busaddr; - frags[0].physaddr += skb->data - cb->data_start; - frags[0].frag_len = skb->len; - frags[0].skb = skb; - frags[0].cmplt = true; - frag_cnt = 1; - break; - } - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - } - } else { - /* no data */ - frags[0].physaddr = cb->busaddr; - frags[0].frag_len = 0; - frags[0].skb = skb; - frags[0].cmplt = true; - frag_cnt = 1; - } - - /* Found one */ - fpdu_info = kzalloc(sizeof(*fpdu_info), GFP_ATOMIC); - if (!fpdu_info) { - rc = -ENOMEM; - goto out; - } - - fpdu_info->cqp_request = nes_get_cqp_request(nesdev); - if (fpdu_info->cqp_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n"); - rc = -ENOMEM; - goto out; - } - - cb = (struct nes_rskb_cb *)&frags[0].skb->cb[0]; - iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - fpdu_info->hdr_len = (((unsigned char *)tcph) + 4 * (tcph->doff)) - cb->data_start; - fpdu_info->data_len = fpdu_len; - tot_len = fpdu_info->hdr_len + fpdu_len - ETH_HLEN; - - if (frags[0].cmplt) { - fpdu_info->hdr_pbase = cb->busaddr; - fpdu_info->hdr_vbase = NULL; - } else { - fpdu_info->hdr_vbase = pci_alloc_consistent(nesdev->pcidev, - fpdu_info->hdr_len, &fpdu_info->hdr_pbase); - if (!fpdu_info->hdr_vbase) { - nes_debug(NES_DBG_PAU, "Unable to allocate memory for pau first frag\n"); - rc = -ENOMEM; - goto out; - } - - /* Copy hdrs, adjusting len and seqnum */ - memcpy(fpdu_info->hdr_vbase, cb->data_start, fpdu_info->hdr_len); - iph = (struct iphdr *)(fpdu_info->hdr_vbase + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - } - - iph->tot_len = cpu_to_be16(tot_len); - iph->saddr = cpu_to_be32(0x7f000001); - - tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt); - tcph->ack_seq = cpu_to_be32(ack); - tcph->window = cpu_to_be16(wnd); - - nesqp->pau_rcv_nxt += fpdu_len + fin_rcvd; - - memcpy(fpdu_info->frags, frags, sizeof(fpdu_info->frags)); - fpdu_info->frag_cnt = frag_cnt; - fpdu_info->nesqp = nesqp; - *pau_fpdu_info = fpdu_info; - - /* Update skb's for next pass */ - for (i = 0; i < frag_cnt; i++) { - cb = (struct nes_rskb_cb *)&frags[i].skb->cb[0]; - skb_pull(frags[i].skb, frags[i].frag_len); - - if (frags[i].skb->len == 0) { - /* Pull skb off the list - it will be freed in the callback */ - if (!skb_queue_empty(&nesqp->pau_list)) - skb_unlink(frags[i].skb, &nesqp->pau_list); - } else { - /* Last skb still has data so update the seq */ - iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt); - } - } - -out: - if (rc) { - if (fpdu_info) { - if (fpdu_info->cqp_request) - nes_put_cqp_request(nesdev, fpdu_info->cqp_request); - kfree(fpdu_info); - } - } - return rc; -} - -/** - * forward_fpdu - send complete fpdus, one at a time - */ -static int forward_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct pau_fpdu_info *fpdu_info; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - u64 u64tmp; - u32 u32tmp; - int rc; - - while (1) { - spin_lock_irqsave(&nesqp->pau_lock, flags); - rc = get_fpdu_info(nesdev, nesqp, &fpdu_info); - if (rc || (fpdu_info == NULL)) { - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - return rc; - } - - cqp_request = fpdu_info->cqp_request; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_OPCODE_IDX, - NES_CQP_DOWNLOAD_SEGMENT | - (((u32)nesvnic->logical_port) << NES_CQP_OP_LOGICAL_PORT_SHIFT)); - - u32tmp = fpdu_info->hdr_len << 16; - u32tmp |= fpdu_info->hdr_len + (u32)fpdu_info->data_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX, - u32tmp); - - u32tmp = (fpdu_info->frags[1].frag_len << 16) | fpdu_info->frags[0].frag_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_2_1_IDX, - u32tmp); - - u32tmp = (fpdu_info->frags[3].frag_len << 16) | fpdu_info->frags[2].frag_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_4_3_IDX, - u32tmp); - - u64tmp = (u64)fpdu_info->hdr_pbase; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX, - lower_32_bits(u64tmp)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_HIGH_IDX, - upper_32_bits(u64tmp)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - lower_32_bits(fpdu_info->frags[0].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_HIGH_IDX, - upper_32_bits(fpdu_info->frags[0].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_LOW_IDX, - lower_32_bits(fpdu_info->frags[1].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_HIGH_IDX, - upper_32_bits(fpdu_info->frags[1].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_LOW_IDX, - lower_32_bits(fpdu_info->frags[2].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_HIGH_IDX, - upper_32_bits(fpdu_info->frags[2].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_LOW_IDX, - lower_32_bits(fpdu_info->frags[3].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_HIGH_IDX, - upper_32_bits(fpdu_info->frags[3].physaddr)); - - cqp_request->cqp_callback_pointer = fpdu_info; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_download_callback; - - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - } - - return 0; -} - -static void process_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - int again = 1; - unsigned long flags; - - do { - /* Ignore rc - if it failed, tcp retries will cause it to try again */ - forward_fpdus(nesvnic, nesqp); - - spin_lock_irqsave(&nesqp->pau_lock, flags); - if (nesqp->pau_pending) { - nesqp->pau_pending = 0; - } else { - nesqp->pau_busy = 0; - again = 0; - } - - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - } while (again); -} - -/** - * queue_fpdus - Handle fpdu's that hw passed up to sw - */ -static void queue_fpdus(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct sk_buff *tmpskb; - struct nes_rskb_cb *cb; - struct iphdr *iph; - struct tcphdr *tcph; - unsigned char *tcph_end; - u32 rcv_nxt; - u32 rcv_wnd; - u32 seqnum; - u32 len; - bool process_it = false; - unsigned long flags; - - /* Move data ptr to after tcp header */ - iph = (struct iphdr *)skb->data; - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - seqnum = be32_to_cpu(tcph->seq); - tcph_end = (((char *)tcph) + (4 * tcph->doff)); - - len = be16_to_cpu(iph->tot_len); - if (skb->len > len) - skb_trim(skb, len); - skb_pull(skb, tcph_end - skb->data); - - /* Initialize tracking values */ - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->seqnum = seqnum; - - /* Make sure data is in the receive window */ - rcv_nxt = nesqp->pau_rcv_nxt; - rcv_wnd = le32_to_cpu(nesqp->nesqp_context->rcv_wnd); - if (!between(seqnum, rcv_nxt, (rcv_nxt + rcv_wnd))) { - nes_mgt_free_skb(nesvnic->nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - return; - } - - spin_lock_irqsave(&nesqp->pau_lock, flags); - - if (nesqp->pau_busy) - nesqp->pau_pending = 1; - else - nesqp->pau_busy = 1; - - /* Queue skb by sequence number */ - if (skb_queue_len(&nesqp->pau_list) == 0) { - __skb_queue_head(&nesqp->pau_list, skb); - } else { - skb_queue_walk(&nesqp->pau_list, tmpskb) { - cb = (struct nes_rskb_cb *)&tmpskb->cb[0]; - if (before(seqnum, cb->seqnum)) - break; - } - __skb_insert(skb, tmpskb->prev, tmpskb, &nesqp->pau_list); - } - if (nesqp->pau_state == PAU_READY) - process_it = true; - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - - if (process_it) - process_fpdus(nesvnic, nesqp); - - return; -} - -/** - * mgt_thread - Handle mgt skbs in a safe context - */ -static int mgt_thread(void *context) -{ - struct nes_vnic *nesvnic = context; - struct sk_buff *skb; - struct nes_rskb_cb *cb; - - while (!kthread_should_stop()) { - wait_event_interruptible(nesvnic->mgt_wait_queue, - skb_queue_len(&nesvnic->mgt_skb_list) || kthread_should_stop()); - while ((skb_queue_len(&nesvnic->mgt_skb_list)) && !kthread_should_stop()) { - skb = skb_dequeue(&nesvnic->mgt_skb_list); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->data_start = skb->data - ETH_HLEN; - cb->busaddr = pci_map_single(nesvnic->nesdev->pcidev, cb->data_start, - nesvnic->max_frame_size, PCI_DMA_TODEVICE); - queue_fpdus(skb, nesvnic, cb->nesqp); - } - } - - /* Closing down so delete any entries on the queue */ - while (skb_queue_len(&nesvnic->mgt_skb_list)) { - skb = skb_dequeue(&nesvnic->mgt_skb_list); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - nes_rem_ref_cm_node(cb->nesqp->cm_node); - dev_kfree_skb_any(skb); - } - return 0; -} - -/** - * nes_queue_skbs - Queue skb so it can be handled in a thread context - */ -void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_rskb_cb *cb; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->nesqp = nesqp; - skb_queue_tail(&nesvnic->mgt_skb_list, skb); - wake_up_interruptible(&nesvnic->mgt_wait_queue); -} - -void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp) -{ - struct sk_buff *skb; - unsigned long flags; - atomic_inc(&pau_qps_destroyed); - - /* Free packets that have not yet been forwarded */ - /* Lock is acquired by skb_dequeue when removing the skb */ - spin_lock_irqsave(&nesqp->pau_lock, flags); - while (skb_queue_len(&nesqp->pau_list)) { - skb = skb_dequeue(&nesqp->pau_list); - nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - } - spin_unlock_irqrestore(&nesqp->pau_lock, flags); -} - -static void nes_chg_qh_handler(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - struct pau_qh_chg *qh_chg = cqp_request->cqp_callback_pointer; - struct nes_cqp_request *new_request; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_adapter *nesadapter; - struct nes_qp *nesqp; - struct nes_v4_quad nes_quad; - u32 crc_value; - u64 u64temp; - - nesadapter = nesdev->nesadapter; - nesqp = qh_chg->nesqp; - - /* Should we handle the bad completion */ - if (cqp_request->major_code) - WARN(1, PFX "Invalid cqp_request major_code=0x%x\n", - cqp_request->major_code); - - switch (nesqp->pau_state) { - case PAU_DEL_QH: - /* Old hash code deleted, now set the new one */ - nesqp->pau_state = PAU_ADD_LB_QH; - new_request = nes_get_cqp_request(nesdev); - if (new_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a new_request.\n"); - WARN_ON(1); - return; - } - - memset(&nes_quad, 0, sizeof(nes_quad)); - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = cpu_to_be32(0x7f000001); - nes_quad.TcpPorts[0] = swab16(nesqp->nesqp_context->tcpPorts[1]); - nes_quad.TcpPorts[1] = swab16(nesqp->nesqp_context->tcpPorts[0]); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_PAU, "new HTE Index = 0x%08X, CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask); - - nesqp->hte_index &= nesadapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - nesqp->nesqp_context->ip0 = cpu_to_le32(0x7f000001); - nesqp->nesqp_context->rcv_nxt = cpu_to_le32(nesqp->pau_rcv_nxt); - - cqp_wqe = &new_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, - NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | - NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - nes_debug(NES_DBG_PAU, "Waiting for CQP completion for adding the quad hash.\n"); - - new_request->cqp_callback_pointer = qh_chg; - new_request->callback = 1; - new_request->cqp_callback = nes_chg_qh_handler; - atomic_set(&new_request->refcount, 1); - nes_post_cqp_request(nesdev, new_request); - break; - - case PAU_ADD_LB_QH: - /* Start processing the queued fpdu's */ - nesqp->pau_state = PAU_READY; - process_fpdus(qh_chg->nesvnic, qh_chg->nesqp); - kfree(qh_chg); - break; - } -} - -/** - * nes_change_quad_hash - */ -static int nes_change_quad_hash(struct nes_device *nesdev, - struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_cqp_request *cqp_request = NULL; - struct pau_qh_chg *qh_chg = NULL; - u64 u64temp; - struct nes_hw_cqp_wqe *cqp_wqe; - int ret = 0; - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n"); - ret = -ENOMEM; - goto chg_qh_err; - } - - qh_chg = kmalloc(sizeof *qh_chg, GFP_ATOMIC); - if (!qh_chg) { - ret = -ENOMEM; - goto chg_qh_err; - } - qh_chg->nesdev = nesdev; - qh_chg->nesvnic = nesvnic; - qh_chg->nesqp = nesqp; - nesqp->pau_state = PAU_DEL_QH; - - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, - NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | NES_CQP_QP_DEL_HTE | - NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - nes_debug(NES_DBG_PAU, "Waiting for CQP completion for deleting the quad hash.\n"); - - cqp_request->cqp_callback_pointer = qh_chg; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_chg_qh_handler; - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); - - return ret; - -chg_qh_err: - kfree(qh_chg); - if (cqp_request) - nes_put_cqp_request(nesdev, cqp_request); - return ret; -} - -/** - * nes_mgt_ce_handler - * This management code deals with any packed and unaligned (pau) fpdu's - * that the hardware cannot handle. - */ -static void nes_mgt_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - struct nes_vnic_mgt *mgtvnic = container_of(cq, struct nes_vnic_mgt, mgt_cq); - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 head; - u32 cq_size; - u32 cqe_count = 0; - u32 cqe_misc; - u32 qp_id = 0; - u32 skbs_needed; - unsigned long context; - struct nes_qp *nesqp; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - - head = cq->cq_head; - cq_size = cq->cq_size; - - while (1) { - cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]); - if (!(cqe_misc & NES_NIC_CQE_VALID)) - break; - - nesqp = NULL; - if (cqe_misc & NES_NIC_CQE_ACCQP_VALID) { - qp_id = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_ACCQP_ID_IDX]); - qp_id &= 0x001fffff; - if (qp_id < nesadapter->max_qp) { - context = (unsigned long)nesadapter->qp_table[qp_id - NES_FIRST_QPN]; - nesqp = (struct nes_qp *)context; - } - } - - if (nesqp) { - if (nesqp->pau_mode == false) { - nesqp->pau_mode = true; /* First time for this qp */ - nesqp->pau_rcv_nxt = le32_to_cpu( - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]); - skb_queue_head_init(&nesqp->pau_list); - spin_lock_init(&nesqp->pau_lock); - atomic_inc(&pau_qps_created); - nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp); - } - - rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail]; - rx_skb->len = 0; - skb_put(rx_skb, cqe_misc & 0x0000ffff); - rx_skb->protocol = eth_type_trans(rx_skb, mgtvnic->nesvnic->netdev); - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, PCI_DMA_FROMDEVICE); - cb->busaddr = 0; - mgtvnic->mgt.rq_tail++; - mgtvnic->mgt.rq_tail &= mgtvnic->mgt.rq_size - 1; - - nes_add_ref_cm_node(nesqp->cm_node); - nes_queue_mgt_skbs(rx_skb, mgtvnic->nesvnic, nesqp); - } else { - printk(KERN_ERR PFX "Invalid QP %d for packed/unaligned handling\n", qp_id); - } - - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0; - cqe_count++; - if (++head >= cq_size) - head = 0; - - if (cqe_count == 255) { - /* Replenish mgt CQ */ - nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (cqe_count << 16)); - nesdev->currcq_count += cqe_count; - cqe_count = 0; - } - - skbs_needed = atomic_inc_return(&mgtvnic->rx_skbs_needed); - if (skbs_needed > (mgtvnic->mgt.rq_size >> 1)) - nes_replenish_mgt_rq(mgtvnic); - } - - cq->cq_head = head; - nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - cq->cq_number | (cqe_count << 16)); - nes_read32(nesdev->regs + NES_CQE_ALLOC); - nesdev->currcq_count += cqe_count; -} - -/** - * nes_init_mgt_qp - */ -int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic) -{ - struct nes_vnic_mgt *mgtvnic; - u32 counter; - void *vmem; - dma_addr_t pmem; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 cqp_head; - unsigned long flags; - struct nes_hw_nic_qp_context *mgt_context; - u64 u64temp; - struct nes_hw_nic_rq_wqe *mgt_rqe; - struct sk_buff *skb; - u32 wqe_count; - struct nes_rskb_cb *cb; - u32 mgt_mem_size; - void *mgt_vbase; - dma_addr_t mgt_pbase; - int i; - int ret; - - /* Allocate space the all mgt QPs once */ - mgtvnic = kcalloc(NES_MGT_QP_COUNT, sizeof(struct nes_vnic_mgt), - GFP_KERNEL); - if (!mgtvnic) - return -ENOMEM; - - /* Allocate fragment, RQ, and CQ; Reuse CEQ based on the PCI function */ - /* We are not sending from this NIC so sq is not allocated */ - mgt_mem_size = 256 + - (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)) + - (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_cqe)) + - sizeof(struct nes_hw_nic_qp_context); - mgt_mem_size = (mgt_mem_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - mgt_vbase = pci_alloc_consistent(nesdev->pcidev, NES_MGT_QP_COUNT * mgt_mem_size, &mgt_pbase); - if (!mgt_vbase) { - kfree(mgtvnic); - nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt host descriptor rings\n"); - return -ENOMEM; - } - - nesvnic->mgt_mem_size = NES_MGT_QP_COUNT * mgt_mem_size; - nesvnic->mgt_vbase = mgt_vbase; - nesvnic->mgt_pbase = mgt_pbase; - - skb_queue_head_init(&nesvnic->mgt_skb_list); - init_waitqueue_head(&nesvnic->mgt_wait_queue); - nesvnic->mgt_thread = kthread_run(mgt_thread, nesvnic, "nes_mgt_thread"); - - for (i = 0; i < NES_MGT_QP_COUNT; i++) { - mgtvnic->nesvnic = nesvnic; - mgtvnic->mgt.qp_id = nesdev->mac_index + NES_MGT_QP_OFFSET + i; - memset(mgt_vbase, 0, mgt_mem_size); - nes_debug(NES_DBG_INIT, "Allocated mgt QP structures at %p (phys = %016lX), size = %u.\n", - mgt_vbase, (unsigned long)mgt_pbase, mgt_mem_size); - - vmem = (void *)(((unsigned long)mgt_vbase + (256 - 1)) & - ~(unsigned long)(256 - 1)); - pmem = (dma_addr_t)(((unsigned long long)mgt_pbase + (256 - 1)) & - ~(unsigned long long)(256 - 1)); - - spin_lock_init(&mgtvnic->mgt.rq_lock); - - /* setup the RQ */ - mgtvnic->mgt.rq_vbase = vmem; - mgtvnic->mgt.rq_pbase = pmem; - mgtvnic->mgt.rq_head = 0; - mgtvnic->mgt.rq_tail = 0; - mgtvnic->mgt.rq_size = NES_MGT_WQ_COUNT; - - /* setup the CQ */ - vmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)); - pmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)); - - mgtvnic->mgt_cq.cq_number = mgtvnic->mgt.qp_id; - mgtvnic->mgt_cq.cq_vbase = vmem; - mgtvnic->mgt_cq.cq_pbase = pmem; - mgtvnic->mgt_cq.cq_head = 0; - mgtvnic->mgt_cq.cq_size = NES_MGT_WQ_COUNT; - - mgtvnic->mgt_cq.ce_handler = nes_mgt_ce_handler; - - /* Send CreateCQ request to CQP */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - cqp_head = nesdev->cqp.sq_head; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - ((u32)mgtvnic->mgt_cq.cq_size << 16)); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32( - mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16)); - u64temp = (u64)mgtvnic->mgt_cq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&mgtvnic->mgt_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* Send CreateQP request to CQP */ - mgt_context = (void *)(&mgtvnic->mgt_cq.cq_vbase[mgtvnic->mgt_cq.cq_size]); - mgt_context->context_words[NES_NIC_CTX_MISC_IDX] = - cpu_to_le32((u32)NES_MGT_CTX_SIZE | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12)); - nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE), - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE)); - if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) - mgt_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE); - - u64temp = (u64)mgtvnic->mgt.rq_pbase; - mgt_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - mgt_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)mgtvnic->mgt.rq_pbase; - mgt_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - mgt_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP | - NES_CQP_QP_TYPE_NIC); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(mgtvnic->mgt.qp_id); - u64temp = (u64)mgtvnic->mgt_cq.cq_pbase + - (mgtvnic->mgt_cq.cq_size * sizeof(struct nes_hw_nic_cqe)); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - nesdev->cqp.sq_head = cqp_head; - - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_INIT, "Waiting for create MGT QP%u to complete.\n", - mgtvnic->mgt.qp_id); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_INIT, "Create MGT QP%u completed, wait_event_timeout ret = %u.\n", - mgtvnic->mgt.qp_id, ret); - if (!ret) { - nes_debug(NES_DBG_INIT, "MGT QP%u create timeout expired\n", mgtvnic->mgt.qp_id); - if (i == 0) { - pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase, - nesvnic->mgt_pbase); - kfree(mgtvnic); - } else { - nes_destroy_mgt(nesvnic); - } - return -EIO; - } - - /* Populate the RQ */ - for (counter = 0; counter < (NES_MGT_WQ_COUNT - 1); counter++) { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (!skb) { - nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name); - return -ENOMEM; - } - - skb->dev = netdev; - - pmem = pci_map_single(nesdev->pcidev, skb->data, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = pmem; - cb->maplen = nesvnic->max_frame_size; - - mgt_rqe = &mgtvnic->mgt.rq_vbase[counter]; - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32((u32)nesvnic->max_frame_size); - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem); - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32)); - mgtvnic->mgt.rx_skb[counter] = skb; - } - - timer_setup(&mgtvnic->rq_wqes_timer, nes_mgt_rq_wqes_timeout, - 0); - - wqe_count = NES_MGT_WQ_COUNT - 1; - mgtvnic->mgt.rq_head = wqe_count; - barrier(); - do { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs + NES_WQE_ALLOC, (counter << 24) | mgtvnic->mgt.qp_id); - } while (wqe_count); - - nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - mgtvnic->mgt_cq.cq_number); - nes_read32(nesdev->regs + NES_CQE_ALLOC); - - mgt_vbase += mgt_mem_size; - mgt_pbase += mgt_mem_size; - nesvnic->mgtvnic[i] = mgtvnic++; - } - return 0; -} - - -void nes_destroy_mgt(struct nes_vnic *nesvnic) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_vnic_mgt *mgtvnic; - struct nes_vnic_mgt *first_mgtvnic; - unsigned long flags; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 cqp_head; - struct sk_buff *rx_skb; - int i; - int ret; - - kthread_stop(nesvnic->mgt_thread); - - /* Free remaining NIC receive buffers */ - first_mgtvnic = nesvnic->mgtvnic[0]; - for (i = 0; i < NES_MGT_QP_COUNT; i++) { - mgtvnic = nesvnic->mgtvnic[i]; - if (mgtvnic == NULL) - continue; - - while (mgtvnic->mgt.rq_head != mgtvnic->mgt.rq_tail) { - rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail]; - nes_mgt_free_skb(nesdev, rx_skb, PCI_DMA_FROMDEVICE); - mgtvnic->mgt.rq_tail++; - mgtvnic->mgt.rq_tail &= (mgtvnic->mgt.rq_size - 1); - } - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy NIC QP */ - cqp_head = nesdev->cqp.sq_head; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - mgtvnic->mgt.qp_id); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - - /* Destroy NIC CQ */ - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_CQ | ((u32)mgtvnic->mgt_cq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16))); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - nesdev->cqp.sq_head = cqp_head; - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u," - " cqp.sq_tail=%u, cqp.sq_size=%u\n", - cqp_head, nesdev->cqp.sq_head, - nesdev->cqp.sq_tail, nesdev->cqp.sq_size); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_SHUTDOWN, "Destroy MGT QP returned, wait_event_timeout ret = %u, cqp_head=%u," - " cqp.sq_head=%u, cqp.sq_tail=%u\n", - ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - if (!ret) - nes_debug(NES_DBG_SHUTDOWN, "MGT QP%u destroy timeout expired\n", - mgtvnic->mgt.qp_id); - - nesvnic->mgtvnic[i] = NULL; - } - - if (nesvnic->mgt_vbase) { - pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase, - nesvnic->mgt_pbase); - nesvnic->mgt_vbase = NULL; - nesvnic->mgt_pbase = 0; - } - - kfree(first_mgtvnic); -} diff --git a/drivers/infiniband/hw/nes/nes_mgt.h b/drivers/infiniband/hw/nes/nes_mgt.h deleted file mode 100644 index 4f7f701c4a81..000000000000 --- a/drivers/infiniband/hw/nes/nes_mgt.h +++ /dev/null @@ -1,97 +0,0 @@ -/* -* Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenIB.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef __NES_MGT_H -#define __NES_MGT_H - -#define MPA_FRAMING 6 /* length is 2 bytes, crc is 4 bytes */ - -int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic); -void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp); -void nes_destroy_mgt(struct nes_vnic *nesvnic); -void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp); - -struct nes_hw_mgt { - struct nes_hw_nic_rq_wqe *rq_vbase; /* virtual address of rq */ - dma_addr_t rq_pbase; /* PCI memory for host rings */ - struct sk_buff *rx_skb[NES_NIC_WQ_SIZE]; - u16 qp_id; - u16 sq_head; - u16 rq_head; - u16 rq_tail; - u16 rq_size; - u8 replenishing_rq; - u8 reserved; - spinlock_t rq_lock; -}; - -struct nes_vnic_mgt { - struct nes_vnic *nesvnic; - struct nes_hw_mgt mgt; - struct nes_hw_nic_cq mgt_cq; - atomic_t rx_skbs_needed; - struct timer_list rq_wqes_timer; - atomic_t rx_skb_timer_running; -}; - -#define MAX_FPDU_FRAGS 4 -struct pau_fpdu_frag { - struct sk_buff *skb; - u64 physaddr; - u32 frag_len; - bool cmplt; -}; - -struct pau_fpdu_info { - struct nes_qp *nesqp; - struct nes_cqp_request *cqp_request; - void *hdr_vbase; - dma_addr_t hdr_pbase; - int hdr_len; - u16 data_len; - u16 frag_cnt; - struct pau_fpdu_frag frags[MAX_FPDU_FRAGS]; -}; - -enum pau_qh_state { - PAU_DEL_QH, - PAU_ADD_LB_QH, - PAU_READY -}; - -struct pau_qh_chg { - struct nes_device *nesdev; - struct nes_vnic *nesvnic; - struct nes_qp *nesqp; -}; - -#endif /* __NES_MGT_H */ diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c deleted file mode 100644 index 16f33454c198..000000000000 --- a/drivers/infiniband/hw/nes/nes_nic.c +++ /dev/null @@ -1,1870 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "nes.h" - -static struct nic_qp_map nic_qp_mapping_0[] = { - {16,0,0,1},{24,4,0,0},{28,8,0,0},{32,12,0,0}, - {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0}, - {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0}, - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_1[] = { - {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0}, - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_2[] = { - {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0} -}; - -static struct nic_qp_map nic_qp_mapping_3[] = { - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_4[] = { - {28,8,0,0},{32,12,0,0} -}; - -static struct nic_qp_map nic_qp_mapping_5[] = { - {29,9,1,0},{33,13,1,0} -}; - -static struct nic_qp_map nic_qp_mapping_6[] = { - {30,10,2,0},{34,14,2,0} -}; - -static struct nic_qp_map nic_qp_mapping_7[] = { - {31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map *nic_qp_mapping_per_function[] = { - nic_qp_mapping_0, nic_qp_mapping_1, nic_qp_mapping_2, nic_qp_mapping_3, - nic_qp_mapping_4, nic_qp_mapping_5, nic_qp_mapping_6, nic_qp_mapping_7 -}; - -static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK - | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN; -static int debug = -1; -static int nics_per_function = 1; - -/** - * nes_netdev_poll - */ -static int nes_netdev_poll(struct napi_struct *napi, int budget) -{ - struct nes_vnic *nesvnic = container_of(napi, struct nes_vnic, napi); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic_cq *nescq = &nesvnic->nic_cq; - - nesvnic->budget = budget; - nescq->cqes_pending = 0; - nescq->rx_cqes_completed = 0; - nescq->cqe_allocs_pending = 0; - nescq->rx_pkts_indicated = 0; - - nes_nic_ce_handler(nesdev, nescq); - - if (nescq->cqes_pending == 0) { - napi_complete(napi); - /* clear out completed cqes and arm */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - nescq->cq_number | (nescq->cqe_allocs_pending << 16)); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - } else { - /* clear out completed cqes but don't arm */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->cq_number | (nescq->cqe_allocs_pending << 16)); - nes_debug(NES_DBG_NETDEV, "%s: exiting with work pending\n", - nesvnic->netdev->name); - } - return nescq->rx_pkts_indicated; -} - - -/** - * nes_netdev_open - Activate the network interface; ifconfig - * ethx up. - */ -static int nes_netdev_open(struct net_device *netdev) -{ - u32 macaddr_low; - u16 macaddr_high; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - int ret; - int i; - struct nes_vnic *first_nesvnic = NULL; - u32 nic_active_bit; - u32 nic_active; - struct list_head *list_pos, *list_temp; - unsigned long flags; - - if (nesvnic->netdev_open == 1) - return 0; - - if (netif_msg_ifup(nesvnic)) - printk(KERN_INFO PFX "%s: enabling interface\n", netdev->name); - - ret = nes_init_nic_qp(nesdev, netdev); - if (ret) { - return ret; - } - - netif_carrier_off(netdev); - netif_stop_queue(netdev); - - if ((!nesvnic->of_device_registered) && (nesvnic->rdma_enabled)) { - nesvnic->nesibdev = nes_init_ofa_device(netdev); - if (nesvnic->nesibdev == NULL) { - printk(KERN_ERR PFX "%s: nesvnic->nesibdev alloc failed", netdev->name); - } else { - nesvnic->nesibdev->nesvnic = nesvnic; - ret = nes_register_ofa_device(nesvnic->nesibdev); - if (ret) { - printk(KERN_ERR PFX "%s: Unable to register RDMA device, ret = %d\n", - netdev->name, ret); - } - } - } - /* Set packet filters */ - nic_active_bit = 1 << nesvnic->nic_index; - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active); - - macaddr_high = ((u16)netdev->dev_addr[0]) << 8; - macaddr_high += (u16)netdev->dev_addr[1]; - - macaddr_low = ((u32)netdev->dev_addr[2]) << 24; - macaddr_low += ((u32)netdev->dev_addr[3]) << 16; - macaddr_low += ((u32)netdev->dev_addr[4]) << 8; - macaddr_low += (u32)netdev->dev_addr[5]; - - /* Program the various MAC regs */ - for (i = 0; i < NES_MAX_PORT_COUNT; i++) { - if (nesvnic->qp_nic_index[i] == 0xf) { - break; - } - nes_debug(NES_DBG_NETDEV, "i=%d, perfect filter table index= %d, PERF FILTER LOW" - " (Addr:%08X) = %08X, HIGH = %08X.\n", - i, nesvnic->qp_nic_index[i], - NES_IDX_PERFECT_FILTER_LOW+ - (nesvnic->qp_nic_index[i] * 8), - macaddr_low, - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8), - macaddr_low); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - } - - - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - nesvnic->nic_cq.cq_number); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) { - first_nesvnic = container_of(list_pos, struct nes_vnic, list); - if (first_nesvnic->netdev_open == 1) - break; - } - if (first_nesvnic->netdev_open == 0) { - nes_debug(NES_DBG_INIT, "Setting up MAC interrupt mask.\n"); - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK + (0x200 * nesdev->mac_index), - ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT | - NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR)); - first_nesvnic = nesvnic; - } - - if (first_nesvnic->linkup) { - /* Enable network packets */ - nesvnic->linkup = 1; - netif_start_queue(netdev); - netif_carrier_on(netdev); - } - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - if (nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_SFP_D) { - nesdev->link_recheck = 1; - mod_delayed_work(system_wq, &nesdev->work, - NES_LINK_RECHECK_DELAY); - } - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - - spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags); - if (nesvnic->of_device_registered) { - nesdev->nesadapter->send_term_ok = 1; - if (nesvnic->linkup == 1) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } else { - nesdev->iw_status = 0; - } - } - spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags); - - napi_enable(&nesvnic->napi); - nesvnic->netdev_open = 1; - - return 0; -} - - -/** - * nes_netdev_stop - */ -static int nes_netdev_stop(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 nic_active_mask; - u32 nic_active; - struct nes_vnic *first_nesvnic = NULL; - struct list_head *list_pos, *list_temp; - unsigned long flags; - - nes_debug(NES_DBG_SHUTDOWN, "nesvnic=%p, nesdev=%p, netdev=%p %s\n", - nesvnic, nesdev, netdev, netdev->name); - if (nesvnic->netdev_open == 0) - return 0; - - if (netif_msg_ifdown(nesvnic)) - printk(KERN_INFO PFX "%s: disabling interface\n", netdev->name); - netif_carrier_off(netdev); - - /* Disable network packets */ - napi_disable(&nesvnic->napi); - netif_stop_queue(netdev); - list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) { - first_nesvnic = container_of(list_pos, struct nes_vnic, list); - if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic)) - break; - } - - if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic) && - (PCI_FUNC(first_nesvnic->nesdev->pcidev->devfn) != - PCI_FUNC(nesvnic->nesdev->pcidev->devfn))) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+ - (0x200*nesdev->mac_index), 0xffffffff); - nes_write_indexed(first_nesvnic->nesdev, - NES_IDX_MAC_INT_MASK+ - (0x200*first_nesvnic->nesdev->mac_index), - ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT | - NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR)); - } else { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+(0x200*nesdev->mac_index), 0xffffffff); - } - - nic_active_mask = ~((u32)(1 << nesvnic->nic_index)); - nes_write_indexed(nesdev, NES_IDX_PERFECT_FILTER_HIGH+ - (nesvnic->perfect_filter_index*8), 0); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active); - - spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags); - if (nesvnic->of_device_registered) { - nesdev->nesadapter->send_term_ok = 0; - nesdev->iw_status = 0; - if (nesvnic->linkup == 1) - nes_port_ibevent(nesvnic); - } - del_timer_sync(&nesvnic->event_timer); - nesvnic->event_timer.function = NULL; - spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags); - - nes_destroy_nic_qp(nesvnic); - - nesvnic->netdev_open = 0; - - return 0; -} - - -/** - * nes_nic_send - */ -static bool nes_nic_send(struct sk_buff *skb, struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic *nesnic = &nesvnic->nic; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct tcphdr *tcph; - __le16 *wqe_fragment_length; - u32 wqe_misc; - u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */ - u16 skb_fragment_index; - dma_addr_t bus_address; - - nic_sqe = &nesnic->sq_vbase[nesnic->sq_head]; - wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - - /* setup the VLAN tag if present */ - if (skb_vlan_tag_present(skb)) { - nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n", - netdev->name, skb_vlan_tag_get(skb)); - wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE; - wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb); - } else - wqe_misc = 0; - - /* bump past the vlan tag */ - wqe_fragment_length++; - /* wqe_fragment_address = (u64 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX]; */ - wqe_misc |= NES_NIC_SQ_WQE_COMPLETION; - - if (skb->ip_summed == CHECKSUM_PARTIAL) { - if (skb_is_gso(skb)) { - tcph = tcp_hdr(skb); - /* nes_debug(NES_DBG_NIC_TX, "%s: TSO request... is_gso = %u seg size = %u\n", - netdev->name, skb_is_gso(skb), skb_shinfo(skb)->gso_size); */ - wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE | (u16)skb_shinfo(skb)->gso_size; - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX, - ((u32)tcph->doff) | - (((u32)(((unsigned char *)tcph) - skb->data)) << 4)); - } - } else { /* CHECKSUM_HW */ - wqe_misc |= NES_NIC_SQ_WQE_DISABLE_CHKSUM; - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX, - skb->len); - memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer, - skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE), skb_headlen(skb))); - wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE), - skb_headlen(skb))); - wqe_fragment_length[1] = 0; - if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) { - if ((skb_shinfo(skb)->nr_frags + 1) > 4) { - nes_debug(NES_DBG_NIC_TX, "%s: Packet with %u fragments not sent, skb_headlen=%u\n", - netdev->name, skb_shinfo(skb)->nr_frags + 2, skb_headlen(skb)); - kfree_skb(skb); - nesvnic->tx_sw_dropped++; - return false; - } - set_bit(nesnic->sq_head, nesnic->first_frag_overflow); - bus_address = pci_map_single(nesdev->pcidev, skb->data + NES_FIRST_FRAG_SIZE, - skb_headlen(skb) - NES_FIRST_FRAG_SIZE, PCI_DMA_TODEVICE); - wqe_fragment_length[wqe_fragment_index++] = - cpu_to_le16(skb_headlen(skb) - NES_FIRST_FRAG_SIZE); - wqe_fragment_length[wqe_fragment_index] = 0; - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - ((u64)(bus_address))); - nesnic->tx_skb[nesnic->sq_head] = skb; - } - - if (skb_headlen(skb) == skb->len) { - if (skb_headlen(skb) <= NES_FIRST_FRAG_SIZE) { - nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_2_1_IDX] = 0; - nesnic->tx_skb[nesnic->sq_head] = skb; - } - } else { - /* Deal with Fragments */ - nesnic->tx_skb[nesnic->sq_head] = skb; - for (skb_fragment_index = 0; skb_fragment_index < skb_shinfo(skb)->nr_frags; - skb_fragment_index++) { - skb_frag_t *frag = - &skb_shinfo(skb)->frags[skb_fragment_index]; - bus_address = skb_frag_dma_map(&nesdev->pcidev->dev, - frag, 0, skb_frag_size(frag), - DMA_TO_DEVICE); - wqe_fragment_length[wqe_fragment_index] = - cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[skb_fragment_index])); - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index), - bus_address); - wqe_fragment_index++; - if (wqe_fragment_index < 5) - wqe_fragment_length[wqe_fragment_index] = 0; - } - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX, wqe_misc); - nesnic->sq_head++; - nesnic->sq_head &= nesnic->sq_size - 1; - return true; -} - - -/** - * nes_netdev_start_xmit - */ -static netdev_tx_t nes_netdev_start_xmit(struct sk_buff *skb, struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic *nesnic = &nesvnic->nic; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct tcphdr *tcph; - /* struct udphdr *udph; */ -#define NES_MAX_TSO_FRAGS MAX_SKB_FRAGS - /* 64K segment plus overflow on each side */ - dma_addr_t tso_bus_address[NES_MAX_TSO_FRAGS]; - dma_addr_t bus_address; - u32 tso_frag_index; - u32 tso_frag_count; - u32 tso_wqe_length; - u32 curr_tcp_seq; - u32 wqe_count=1; - struct iphdr *iph; - __le16 *wqe_fragment_length; - u32 nr_frags; - u32 original_first_length; - /* u64 *wqe_fragment_address; */ - /* first fragment (0) is used by copy buffer */ - u16 wqe_fragment_index=1; - u16 hoffset; - u16 nhoffset; - u16 wqes_needed; - u16 wqes_available; - u32 wqe_misc; - - /* - * nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u," - * " (%u frags), tso_size=%u\n", - * netdev->name, skb->len, skb_headlen(skb), - * skb_shinfo(skb)->nr_frags, skb_is_gso(skb)); - */ - - if (netif_queue_stopped(netdev)) - return NETDEV_TX_BUSY; - - /* Check if SQ is full */ - if ((((nesnic->sq_tail+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) == 1) { - if (!netif_queue_stopped(netdev)) { - netif_stop_queue(netdev); - barrier(); - if ((((((volatile u16)nesnic->sq_tail)+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) != 1) { - netif_start_queue(netdev); - goto sq_no_longer_full; - } - } - nesvnic->sq_full++; - return NETDEV_TX_BUSY; - } - -sq_no_longer_full: - nr_frags = skb_shinfo(skb)->nr_frags; - if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) { - nr_frags++; - } - /* Check if too many fragments */ - if (unlikely((nr_frags > 4))) { - if (skb_is_gso(skb)) { - nesvnic->segmented_tso_requests++; - nesvnic->tso_requests++; - /* Basically 4 fragments available per WQE with extended fragments */ - wqes_needed = nr_frags >> 2; - wqes_needed += (nr_frags&3)?1:0; - wqes_available = (((nesnic->sq_tail+nesnic->sq_size)-nesnic->sq_head) - 1) & - (nesnic->sq_size - 1); - - if (unlikely(wqes_needed > wqes_available)) { - if (!netif_queue_stopped(netdev)) { - netif_stop_queue(netdev); - barrier(); - wqes_available = (((((volatile u16)nesnic->sq_tail)+nesnic->sq_size)-nesnic->sq_head) - 1) & - (nesnic->sq_size - 1); - if (wqes_needed <= wqes_available) { - netif_start_queue(netdev); - goto tso_sq_no_longer_full; - } - } - nesvnic->sq_full++; - nes_debug(NES_DBG_NIC_TX, "%s: HNIC SQ full- TSO request has too many frags!\n", - netdev->name); - return NETDEV_TX_BUSY; - } -tso_sq_no_longer_full: - /* Map all the buffers */ - for (tso_frag_count=0; tso_frag_count < skb_shinfo(skb)->nr_frags; - tso_frag_count++) { - skb_frag_t *frag = - &skb_shinfo(skb)->frags[tso_frag_count]; - tso_bus_address[tso_frag_count] = - skb_frag_dma_map(&nesdev->pcidev->dev, - frag, 0, skb_frag_size(frag), - DMA_TO_DEVICE); - } - - tso_frag_index = 0; - curr_tcp_seq = ntohl(tcp_hdr(skb)->seq); - hoffset = skb_transport_header(skb) - skb->data; - nhoffset = skb_network_header(skb) - skb->data; - original_first_length = hoffset + ((((struct tcphdr *)skb_transport_header(skb))->doff)<<2); - - for (wqe_count=0; wqe_count<((u32)wqes_needed); wqe_count++) { - tso_wqe_length = 0; - nic_sqe = &nesnic->sq_vbase[nesnic->sq_head]; - wqe_fragment_length = - (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* setup the VLAN tag if present */ - if (skb_vlan_tag_present(skb)) { - nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n", - netdev->name, - skb_vlan_tag_get(skb)); - wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE; - wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb); - } else - wqe_misc = 0; - - /* bump past the vlan tag */ - wqe_fragment_length++; - - /* Assumes header totally fits in allocated buffer and is in first fragment */ - if (original_first_length > NES_FIRST_FRAG_SIZE) { - nes_debug(NES_DBG_NIC_TX, "ERROR: SKB header too big, headlen=%u, FIRST_FRAG_SIZE=%u\n", - original_first_length, NES_FIRST_FRAG_SIZE); - nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u," - " (%u frags), is_gso = %u tso_size=%u\n", - netdev->name, - skb->len, skb_headlen(skb), - skb_shinfo(skb)->nr_frags, skb_is_gso(skb), skb_shinfo(skb)->gso_size); - } - memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer, - skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE), - original_first_length)); - iph = (struct iphdr *) - (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[nhoffset]); - tcph = (struct tcphdr *) - (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[hoffset]); - if ((wqe_count+1)!=(u32)wqes_needed) { - tcph->fin = 0; - tcph->psh = 0; - tcph->rst = 0; - tcph->urg = 0; - } - if (wqe_count) { - tcph->syn = 0; - } - tcph->seq = htonl(curr_tcp_seq); - wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE), - original_first_length)); - - wqe_fragment_index = 1; - if ((wqe_count==0) && (skb_headlen(skb) > original_first_length)) { - set_bit(nesnic->sq_head, nesnic->first_frag_overflow); - bus_address = pci_map_single(nesdev->pcidev, skb->data + original_first_length, - skb_headlen(skb) - original_first_length, PCI_DMA_TODEVICE); - wqe_fragment_length[wqe_fragment_index++] = - cpu_to_le16(skb_headlen(skb) - original_first_length); - wqe_fragment_length[wqe_fragment_index] = 0; - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - bus_address); - tso_wqe_length += skb_headlen(skb) - - original_first_length; - } - while (wqe_fragment_index < 5) { - wqe_fragment_length[wqe_fragment_index] = - cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index])); - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index), - (u64)tso_bus_address[tso_frag_index]); - wqe_fragment_index++; - tso_wqe_length += skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index++]); - if (wqe_fragment_index < 5) - wqe_fragment_length[wqe_fragment_index] = 0; - if (tso_frag_index == tso_frag_count) - break; - } - if ((wqe_count+1) == (u32)wqes_needed) { - nesnic->tx_skb[nesnic->sq_head] = skb; - } else { - nesnic->tx_skb[nesnic->sq_head] = NULL; - } - wqe_misc |= NES_NIC_SQ_WQE_COMPLETION | (u16)skb_shinfo(skb)->gso_size; - if ((tso_wqe_length + original_first_length) > skb_shinfo(skb)->gso_size) { - wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE; - } else { - iph->tot_len = htons(tso_wqe_length + original_first_length - nhoffset); - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX, - wqe_misc); - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX, - ((u32)tcph->doff) | (((u32)hoffset) << 4)); - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX, - tso_wqe_length + original_first_length); - curr_tcp_seq += tso_wqe_length; - nesnic->sq_head++; - nesnic->sq_head &= nesnic->sq_size-1; - } - } else { - hoffset = skb_transport_header(skb) - skb->data; - nhoffset = skb_network_header(skb) - skb->data; - if (skb_linearize(skb)) { - nesvnic->tx_sw_dropped++; - kfree_skb(skb); - return NETDEV_TX_OK; - } - nesvnic->linearized_skbs++; - skb_set_transport_header(skb, hoffset); - skb_set_network_header(skb, nhoffset); - if (!nes_nic_send(skb, netdev)) - return NETDEV_TX_OK; - } - } else { - if (!nes_nic_send(skb, netdev)) - return NETDEV_TX_OK; - } - - barrier(); - - if (wqe_count) - nes_write32(nesdev->regs+NES_WQE_ALLOC, - (wqe_count << 24) | (1 << 23) | nesvnic->nic.qp_id); - - netif_trans_update(netdev); - - return NETDEV_TX_OK; -} - - -/** - * nes_netdev_get_stats - */ -static struct net_device_stats *nes_netdev_get_stats(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u64 u64temp; - u32 u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD + (nesvnic->nic_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->endnode_nstat_rx_discard += u32temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_rx_octets += u64temp; - nesvnic->netstats.rx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_rx_frames += u64temp; - nesvnic->netstats.rx_packets += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_tx_octets += u64temp; - nesvnic->netstats.tx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_tx_frames += u64temp; - nesvnic->netstats.tx_packets += u64temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_short_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_oversized_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_jabber_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_length_errors += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_crc_errors += u32temp; - nesvnic->netstats.rx_crc_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_tx_errors += u32temp; - nesvnic->netstats.tx_errors += u32temp; - - return &nesvnic->netstats; -} - - -/** - * nes_netdev_tx_timeout - */ -static void nes_netdev_tx_timeout(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - if (netif_msg_timer(nesvnic)) - nes_debug(NES_DBG_NIC_TX, "%s: tx timeout\n", netdev->name); -} - - -/** - * nes_netdev_set_mac_address - */ -static int nes_netdev_set_mac_address(struct net_device *netdev, void *p) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct sockaddr *mac_addr = p; - int i; - u32 macaddr_low; - u16 macaddr_high; - - if (!is_valid_ether_addr(mac_addr->sa_data)) - return -EADDRNOTAVAIL; - - memcpy(netdev->dev_addr, mac_addr->sa_data, netdev->addr_len); - printk(PFX "%s: Address length = %d, Address = %pM\n", - __func__, netdev->addr_len, mac_addr->sa_data); - macaddr_high = ((u16)netdev->dev_addr[0]) << 8; - macaddr_high += (u16)netdev->dev_addr[1]; - macaddr_low = ((u32)netdev->dev_addr[2]) << 24; - macaddr_low += ((u32)netdev->dev_addr[3]) << 16; - macaddr_low += ((u32)netdev->dev_addr[4]) << 8; - macaddr_low += (u32)netdev->dev_addr[5]; - - for (i = 0; i < NES_MAX_PORT_COUNT; i++) { - if (nesvnic->qp_nic_index[i] == 0xf) { - break; - } - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8), - macaddr_low); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - } - return 0; -} - - -static void set_allmulti(struct nes_device *nesdev, u32 nic_active_bit) -{ - u32 nic_active; - - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); -} - -#define get_addr(addrs, index) ((addrs) + (index) * ETH_ALEN) - -/** - * nes_netdev_set_multicast_list - */ -static void nes_netdev_set_multicast_list(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - u32 nic_active_bit; - u32 nic_active; - u32 perfect_filter_register_address; - u32 macaddr_low; - u16 macaddr_high; - u8 mc_all_on = 0; - u8 mc_index; - int mc_nic_index = -1; - u8 pft_entries_preallocated = max(nesadapter->adapter_fcn_count * - nics_per_function, 4); - u8 max_pft_entries_avaiable = NES_PFT_SIZE - pft_entries_preallocated; - unsigned long flags; - int mc_count = netdev_mc_count(netdev); - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - nic_active_bit = 1 << nesvnic->nic_index; - - if (netdev->flags & IFF_PROMISC) { - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - mc_all_on = 1; - } else if ((netdev->flags & IFF_ALLMULTI) || - (nesvnic->nic_index > 3)) { - set_allmulti(nesdev, nic_active_bit); - mc_all_on = 1; - } else { - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - } - - nes_debug(NES_DBG_NIC_RX, "Number of MC entries = %d, Promiscuous = %d, All Multicast = %d.\n", - mc_count, !!(netdev->flags & IFF_PROMISC), - !!(netdev->flags & IFF_ALLMULTI)); - if (!mc_all_on) { - char *addrs; - int i; - struct netdev_hw_addr *ha; - - addrs = kmalloc_array(mc_count, ETH_ALEN, GFP_ATOMIC); - if (!addrs) { - set_allmulti(nesdev, nic_active_bit); - goto unlock; - } - i = 0; - netdev_for_each_mc_addr(ha, netdev) - memcpy(get_addr(addrs, i++), ha->addr, ETH_ALEN); - - perfect_filter_register_address = NES_IDX_PERFECT_FILTER_LOW + - pft_entries_preallocated * 0x8; - for (i = 0, mc_index = 0; mc_index < max_pft_entries_avaiable; - mc_index++) { - while (i < mc_count && nesvnic->mcrq_mcast_filter && - ((mc_nic_index = nesvnic->mcrq_mcast_filter(nesvnic, - get_addr(addrs, i++))) == 0)); - if (mc_nic_index < 0) - mc_nic_index = nesvnic->nic_index; - while (nesadapter->pft_mcast_map[mc_index] < 16 && - nesadapter->pft_mcast_map[mc_index] != - nesvnic->nic_index && - mc_index < max_pft_entries_avaiable) { - nes_debug(NES_DBG_NIC_RX, - "mc_index=%d skipping nic_index=%d, used for=%d\n", - mc_index, nesvnic->nic_index, - nesadapter->pft_mcast_map[mc_index]); - mc_index++; - } - if (mc_index >= max_pft_entries_avaiable) - break; - if (i < mc_count) { - char *addr = get_addr(addrs, i++); - - nes_debug(NES_DBG_NIC_RX, "Assigning MC Address %pM to register 0x%04X nic_idx=%d\n", - addr, - perfect_filter_register_address+(mc_index * 8), - mc_nic_index); - macaddr_high = ((u8) addr[0]) << 8; - macaddr_high += (u8) addr[1]; - macaddr_low = ((u8) addr[2]) << 24; - macaddr_low += ((u8) addr[3]) << 16; - macaddr_low += ((u8) addr[4]) << 8; - macaddr_low += (u8) addr[5]; - - nes_write_indexed(nesdev, - perfect_filter_register_address+(mc_index * 8), - macaddr_low); - nes_write_indexed(nesdev, - perfect_filter_register_address+4+(mc_index * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)(1<pft_mcast_map[mc_index] = - nesvnic->nic_index; - } else { - nes_debug(NES_DBG_NIC_RX, "Clearing MC Address at register 0x%04X\n", - perfect_filter_register_address+(mc_index * 8)); - nes_write_indexed(nesdev, - perfect_filter_register_address+4+(mc_index * 8), - 0); - nesadapter->pft_mcast_map[mc_index] = 255; - } - } - kfree(addrs); - /* PFT is not large enough */ - if (i < mc_count) - set_allmulti(nesdev, nic_active_bit); - } - -unlock: - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); -} - - -/** - * nes_netdev_change_mtu - */ -static int nes_netdev_change_mtu(struct net_device *netdev, int new_mtu) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u8 jumbomode = 0; - u32 nic_active; - u32 nic_active_bit; - u32 uc_all_active; - u32 mc_all_active; - - netdev->mtu = new_mtu; - nesvnic->max_frame_size = new_mtu + VLAN_ETH_HLEN; - - if (netdev->mtu > ETH_DATA_LEN) { - jumbomode=1; - } - nes_nic_init_timer_defaults(nesdev, jumbomode); - - if (netif_running(netdev)) { - nic_active_bit = 1 << nesvnic->nic_index; - mc_all_active = nes_read_indexed(nesdev, - NES_IDX_NIC_MULTICAST_ALL) & nic_active_bit; - uc_all_active = nes_read_indexed(nesdev, - NES_IDX_NIC_UNICAST_ALL) & nic_active_bit; - - nes_netdev_stop(netdev); - nes_netdev_open(netdev); - - nic_active = nes_read_indexed(nesdev, - NES_IDX_NIC_MULTICAST_ALL); - nic_active |= mc_all_active; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, - nic_active); - - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active |= uc_all_active; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - } - - return 0; -} - - -static const char nes_ethtool_stringset[][ETH_GSTRING_LEN] = { - "Link Change Interrupts", - "Linearized SKBs", - "T/GSO Requests", - "Pause Frames Sent", - "Pause Frames Received", - "Internal Routing Errors", - "SQ SW Dropped SKBs", - "SQ Full", - "Segmented TSO Requests", - "Rx Symbol Errors", - "Rx Jabber Errors", - "Rx Oversized Frames", - "Rx Short Frames", - "Rx Length Errors", - "Rx CRC Errors", - "Rx Port Discard", - "Endnode Rx Discards", - "Endnode Rx Octets", - "Endnode Rx Frames", - "Endnode Tx Octets", - "Endnode Tx Frames", - "Tx Errors", - "mh detected", - "mh pauses", - "Retransmission Count", - "CM Connects", - "CM Accepts", - "Disconnects", - "Connected Events", - "Connect Requests", - "CM Rejects", - "ModifyQP Timeouts", - "CreateQPs", - "SW DestroyQPs", - "DestroyQPs", - "CM Closes", - "CM Packets Sent", - "CM Packets Bounced", - "CM Packets Created", - "CM Packets Rcvd", - "CM Packets Dropped", - "CM Packets Retrans", - "CM Listens Created", - "CM Listens Destroyed", - "CM Backlog Drops", - "CM Loopbacks", - "CM Nodes Created", - "CM Nodes Destroyed", - "CM Accel Drops", - "CM Resets Received", - "Free 4Kpbls", - "Free 256pbls", - "Timer Inits", - "PAU CreateQPs", - "PAU DestroyQPs", -}; -#define NES_ETHTOOL_STAT_COUNT ARRAY_SIZE(nes_ethtool_stringset) - - -/** - * nes_netdev_get_sset_count - */ -static int nes_netdev_get_sset_count(struct net_device *netdev, int stringset) -{ - if (stringset == ETH_SS_STATS) - return NES_ETHTOOL_STAT_COUNT; - else - return -EINVAL; -} - - -/** - * nes_netdev_get_strings - */ -static void nes_netdev_get_strings(struct net_device *netdev, u32 stringset, - u8 *ethtool_strings) -{ - if (stringset == ETH_SS_STATS) - memcpy(ethtool_strings, - &nes_ethtool_stringset, - sizeof(nes_ethtool_stringset)); -} - - -/** - * nes_netdev_get_ethtool_stats - */ - -static void nes_netdev_get_ethtool_stats(struct net_device *netdev, - struct ethtool_stats *target_ethtool_stats, u64 *target_stat_values) -{ - u64 u64temp; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 nic_count; - u32 u32temp; - u32 index = 0; - - target_ethtool_stats->n_stats = NES_ETHTOOL_STAT_COUNT; - target_stat_values[index] = nesvnic->nesdev->link_status_interrupts; - target_stat_values[++index] = nesvnic->linearized_skbs; - target_stat_values[++index] = nesvnic->tso_requests; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_pause_frames_sent += u32temp; - target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_sent; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_pause_frames_received += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PORT_RX_DISCARDS + (nesvnic->nesdev->mac_index*0x40)); - nesvnic->nesdev->port_rx_discards += u32temp; - nesvnic->netstats.rx_dropped += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PORT_TX_DISCARDS + (nesvnic->nesdev->mac_index*0x40)); - nesvnic->nesdev->port_tx_discards += u32temp; - nesvnic->netstats.tx_dropped += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_short_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_oversized_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_jabber_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_length_errors += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_crc_errors += u32temp; - nesvnic->netstats.rx_crc_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_tx_errors += u32temp; - nesvnic->netstats.tx_errors += u32temp; - - for (nic_count = 0; nic_count < NES_MAX_PORT_COUNT; nic_count++) { - if (nesvnic->qp_nic_index[nic_count] == 0xf) - break; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD + - (nesvnic->qp_nic_index[nic_count]*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->endnode_nstat_rx_discard += u32temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_rx_octets += u64temp; - nesvnic->netstats.rx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_rx_frames += u64temp; - nesvnic->netstats.rx_packets += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_tx_octets += u64temp; - nesvnic->netstats.tx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_tx_frames += u64temp; - nesvnic->netstats.tx_packets += u64temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_IPV4_TCP_REXMITS + (nesvnic->qp_nic_index[nic_count]*0x200)); - nesvnic->endnode_ipv4_tcp_retransmits += u32temp; - } - - target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_received; - target_stat_values[++index] = nesdev->nesadapter->nic_rx_eth_route_err; - target_stat_values[++index] = nesvnic->tx_sw_dropped; - target_stat_values[++index] = nesvnic->sq_full; - target_stat_values[++index] = nesvnic->segmented_tso_requests; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_symbol_err_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_jabber_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_oversized_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_short_frames; - target_stat_values[++index] = nesvnic->netstats.rx_length_errors; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_crc_errors; - target_stat_values[++index] = nesvnic->nesdev->port_rx_discards; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_discard; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_octets; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_frames; - target_stat_values[++index] = nesvnic->endnode_nstat_tx_octets; - target_stat_values[++index] = nesvnic->endnode_nstat_tx_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_tx_errors; - target_stat_values[++index] = mh_detected; - target_stat_values[++index] = mh_pauses_sent; - target_stat_values[++index] = nesvnic->endnode_ipv4_tcp_retransmits; - target_stat_values[++index] = atomic_read(&cm_connects); - target_stat_values[++index] = atomic_read(&cm_accepts); - target_stat_values[++index] = atomic_read(&cm_disconnects); - target_stat_values[++index] = atomic_read(&cm_connecteds); - target_stat_values[++index] = atomic_read(&cm_connect_reqs); - target_stat_values[++index] = atomic_read(&cm_rejects); - target_stat_values[++index] = atomic_read(&mod_qp_timouts); - target_stat_values[++index] = atomic_read(&qps_created); - target_stat_values[++index] = atomic_read(&sw_qps_destroyed); - target_stat_values[++index] = atomic_read(&qps_destroyed); - target_stat_values[++index] = atomic_read(&cm_closes); - target_stat_values[++index] = cm_packets_sent; - target_stat_values[++index] = cm_packets_bounced; - target_stat_values[++index] = cm_packets_created; - target_stat_values[++index] = cm_packets_received; - target_stat_values[++index] = cm_packets_dropped; - target_stat_values[++index] = cm_packets_retrans; - target_stat_values[++index] = atomic_read(&cm_listens_created); - target_stat_values[++index] = atomic_read(&cm_listens_destroyed); - target_stat_values[++index] = cm_backlog_drops; - target_stat_values[++index] = atomic_read(&cm_loopbacks); - target_stat_values[++index] = atomic_read(&cm_nodes_created); - target_stat_values[++index] = atomic_read(&cm_nodes_destroyed); - target_stat_values[++index] = atomic_read(&cm_accel_dropped_pkts); - target_stat_values[++index] = atomic_read(&cm_resets_recvd); - target_stat_values[++index] = nesadapter->free_4kpbl; - target_stat_values[++index] = nesadapter->free_256pbl; - target_stat_values[++index] = int_mod_timer_init; - target_stat_values[++index] = atomic_read(&pau_qps_created); - target_stat_values[++index] = atomic_read(&pau_qps_destroyed); -} - -/** - * nes_netdev_get_drvinfo - */ -static void nes_netdev_get_drvinfo(struct net_device *netdev, - struct ethtool_drvinfo *drvinfo) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(nesvnic->nesdev->pcidev), - sizeof(drvinfo->bus_info)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%u.%u", nesadapter->firmware_version >> 16, - nesadapter->firmware_version & 0x000000ff); - strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); -} - - -/** - * nes_netdev_set_coalesce - */ -static int nes_netdev_set_coalesce(struct net_device *netdev, - struct ethtool_coalesce *et_coalesce) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - unsigned long flags; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - if (et_coalesce->rx_max_coalesced_frames_low) { - shared_timer->threshold_low = et_coalesce->rx_max_coalesced_frames_low; - } - if (et_coalesce->rx_max_coalesced_frames_irq) { - shared_timer->threshold_target = et_coalesce->rx_max_coalesced_frames_irq; - } - if (et_coalesce->rx_max_coalesced_frames_high) { - shared_timer->threshold_high = et_coalesce->rx_max_coalesced_frames_high; - } - if (et_coalesce->rx_coalesce_usecs_low) { - shared_timer->timer_in_use_min = et_coalesce->rx_coalesce_usecs_low; - } - if (et_coalesce->rx_coalesce_usecs_high) { - shared_timer->timer_in_use_max = et_coalesce->rx_coalesce_usecs_high; - } - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - - /* using this to drive total interrupt moderation */ - nesadapter->et_rx_coalesce_usecs_irq = et_coalesce->rx_coalesce_usecs_irq; - if (et_coalesce->use_adaptive_rx_coalesce) { - nesadapter->et_use_adaptive_rx_coalesce = 1; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC; - nesadapter->et_rx_coalesce_usecs_irq = 0; - if (et_coalesce->pkt_rate_low) { - nesadapter->et_pkt_rate_low = et_coalesce->pkt_rate_low; - } - } else { - nesadapter->et_use_adaptive_rx_coalesce = 0; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT; - if (nesadapter->et_rx_coalesce_usecs_irq) { - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, - 0x80000000 | ((u32)(nesadapter->et_rx_coalesce_usecs_irq*8))); - } - } - return 0; -} - - -/** - * nes_netdev_get_coalesce - */ -static int nes_netdev_get_coalesce(struct net_device *netdev, - struct ethtool_coalesce *et_coalesce) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct ethtool_coalesce temp_et_coalesce; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - unsigned long flags; - - memset(&temp_et_coalesce, 0, sizeof(temp_et_coalesce)); - temp_et_coalesce.rx_coalesce_usecs_irq = nesadapter->et_rx_coalesce_usecs_irq; - temp_et_coalesce.use_adaptive_rx_coalesce = nesadapter->et_use_adaptive_rx_coalesce; - temp_et_coalesce.rate_sample_interval = nesadapter->et_rate_sample_interval; - temp_et_coalesce.pkt_rate_low = nesadapter->et_pkt_rate_low; - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - temp_et_coalesce.rx_max_coalesced_frames_low = shared_timer->threshold_low; - temp_et_coalesce.rx_max_coalesced_frames_irq = shared_timer->threshold_target; - temp_et_coalesce.rx_max_coalesced_frames_high = shared_timer->threshold_high; - temp_et_coalesce.rx_coalesce_usecs_low = shared_timer->timer_in_use_min; - temp_et_coalesce.rx_coalesce_usecs_high = shared_timer->timer_in_use_max; - if (nesadapter->et_use_adaptive_rx_coalesce) { - temp_et_coalesce.rx_coalesce_usecs_irq = shared_timer->timer_in_use; - } - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - memcpy(et_coalesce, &temp_et_coalesce, sizeof(*et_coalesce)); - return 0; -} - - -/** - * nes_netdev_get_pauseparam - */ -static void nes_netdev_get_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *et_pauseparam) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - et_pauseparam->autoneg = 0; - et_pauseparam->rx_pause = (nesvnic->nesdev->disable_rx_flow_control == 0) ? 1:0; - et_pauseparam->tx_pause = (nesvnic->nesdev->disable_tx_flow_control == 0) ? 1:0; -} - - -/** - * nes_netdev_set_pauseparam - */ -static int nes_netdev_set_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *et_pauseparam) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 u32temp; - - if (et_pauseparam->autoneg) { - /* TODO: should return unsupported */ - return 0; - } - if ((et_pauseparam->tx_pause == 1) && (nesdev->disable_tx_flow_control == 1)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); - u32temp |= NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); - nesdev->disable_tx_flow_control = 0; - } else if ((et_pauseparam->tx_pause == 0) && (nesdev->disable_tx_flow_control == 0)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); - u32temp &= ~NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); - nesdev->disable_tx_flow_control = 1; - } - if ((et_pauseparam->rx_pause == 1) && (nesdev->disable_rx_flow_control == 1)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40)); - u32temp &= ~NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp); - nesdev->disable_rx_flow_control = 0; - } else if ((et_pauseparam->rx_pause == 0) && (nesdev->disable_rx_flow_control == 0)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40)); - u32temp |= NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp); - nesdev->disable_rx_flow_control = 1; - } - - return 0; -} - - -/** - * nes_netdev_get_settings - */ -static int nes_netdev_get_link_ksettings(struct net_device *netdev, - struct ethtool_link_ksettings *cmd) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 mac_index = nesdev->mac_index; - u8 phy_type = nesadapter->phy_type[mac_index]; - u8 phy_index = nesadapter->phy_index[mac_index]; - u16 phy_data; - u32 supported, advertising; - - cmd->base.duplex = DUPLEX_FULL; - cmd->base.port = PORT_MII; - - if (nesadapter->OneG_Mode) { - cmd->base.speed = SPEED_1000; - if (phy_type == NES_PHY_TYPE_PUMA_1G) { - supported = SUPPORTED_1000baseT_Full; - advertising = ADVERTISED_1000baseT_Full; - cmd->base.autoneg = AUTONEG_DISABLE; - cmd->base.phy_address = mac_index; - } else { - unsigned long flags; - - supported = SUPPORTED_1000baseT_Full - | SUPPORTED_Autoneg; - advertising = ADVERTISED_1000baseT_Full - | ADVERTISED_Autoneg; - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - if (phy_data & 0x1000) - cmd->base.autoneg = AUTONEG_ENABLE; - else - cmd->base.autoneg = AUTONEG_DISABLE; - cmd->base.phy_address = phy_index; - } - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.supported, supported); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.advertising, advertising); - return 0; - } - if ((phy_type == NES_PHY_TYPE_ARGUS) || - (phy_type == NES_PHY_TYPE_SFP_D) || - (phy_type == NES_PHY_TYPE_KR)) { - cmd->base.port = PORT_FIBRE; - supported = SUPPORTED_FIBRE; - advertising = ADVERTISED_FIBRE; - cmd->base.phy_address = phy_index; - } else { - supported = SUPPORTED_10000baseT_Full; - advertising = ADVERTISED_10000baseT_Full; - cmd->base.phy_address = mac_index; - } - cmd->base.speed = SPEED_10000; - cmd->base.autoneg = AUTONEG_DISABLE; - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, - supported); - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, - advertising); - - return 0; -} - - -/** - * nes_netdev_set_settings - */ -static int -nes_netdev_set_link_ksettings(struct net_device *netdev, - const struct ethtool_link_ksettings *cmd) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[nesdev->mac_index] != NES_PHY_TYPE_PUMA_1G)) { - unsigned long flags; - u16 phy_data; - u8 phy_index = nesadapter->phy_index[nesdev->mac_index]; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - if (cmd->base.autoneg) { - /* Turn on Full duplex, Autoneg, and restart autonegotiation */ - phy_data |= 0x1300; - } else { - /* Turn off autoneg */ - phy_data &= ~0x1000; - } - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - } - - return 0; -} - - -static const struct ethtool_ops nes_ethtool_ops = { - .get_link = ethtool_op_get_link, - .get_strings = nes_netdev_get_strings, - .get_sset_count = nes_netdev_get_sset_count, - .get_ethtool_stats = nes_netdev_get_ethtool_stats, - .get_drvinfo = nes_netdev_get_drvinfo, - .get_coalesce = nes_netdev_get_coalesce, - .set_coalesce = nes_netdev_set_coalesce, - .get_pauseparam = nes_netdev_get_pauseparam, - .set_pauseparam = nes_netdev_set_pauseparam, - .get_link_ksettings = nes_netdev_get_link_ksettings, - .set_link_ksettings = nes_netdev_set_link_ksettings, -}; - -static void nes_vlan_mode(struct net_device *netdev, struct nes_device *nesdev, netdev_features_t features) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 u32temp; - unsigned long flags; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - - nes_debug(NES_DBG_NETDEV, "%s: %s\n", __func__, netdev->name); - - /* Enable/Disable VLAN Stripping */ - u32temp = nes_read_indexed(nesdev, NES_IDX_PCIX_DIAG); - if (features & NETIF_F_HW_VLAN_CTAG_RX) - u32temp &= 0xfdffffff; - else - u32temp |= 0x02000000; - - nes_write_indexed(nesdev, NES_IDX_PCIX_DIAG, u32temp); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); -} - -static netdev_features_t nes_fix_features(struct net_device *netdev, netdev_features_t features) -{ - /* - * Since there is no support for separate rx/tx vlan accel - * enable/disable make sure tx flag is always in same state as rx. - */ - if (features & NETIF_F_HW_VLAN_CTAG_RX) - features |= NETIF_F_HW_VLAN_CTAG_TX; - else - features &= ~NETIF_F_HW_VLAN_CTAG_TX; - - return features; -} - -static int nes_set_features(struct net_device *netdev, netdev_features_t features) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 changed = netdev->features ^ features; - - if (changed & NETIF_F_HW_VLAN_CTAG_RX) - nes_vlan_mode(netdev, nesdev, features); - - return 0; -} - -static const struct net_device_ops nes_netdev_ops = { - .ndo_open = nes_netdev_open, - .ndo_stop = nes_netdev_stop, - .ndo_start_xmit = nes_netdev_start_xmit, - .ndo_get_stats = nes_netdev_get_stats, - .ndo_tx_timeout = nes_netdev_tx_timeout, - .ndo_set_mac_address = nes_netdev_set_mac_address, - .ndo_set_rx_mode = nes_netdev_set_multicast_list, - .ndo_change_mtu = nes_netdev_change_mtu, - .ndo_validate_addr = eth_validate_addr, - .ndo_fix_features = nes_fix_features, - .ndo_set_features = nes_set_features, -}; - -/** - * nes_netdev_init - initialize network device - */ -struct net_device *nes_netdev_init(struct nes_device *nesdev, - void __iomem *mmio_addr) -{ - u64 u64temp; - struct nes_vnic *nesvnic; - struct net_device *netdev; - struct nic_qp_map *curr_qp_map; - u8 phy_type = nesdev->nesadapter->phy_type[nesdev->mac_index]; - - netdev = alloc_etherdev(sizeof(struct nes_vnic)); - if (!netdev) { - printk(KERN_ERR PFX "nesvnic etherdev alloc failed"); - return NULL; - } - nesvnic = netdev_priv(netdev); - - nes_debug(NES_DBG_INIT, "netdev = %p, %s\n", netdev, netdev->name); - - SET_NETDEV_DEV(netdev, &nesdev->pcidev->dev); - - netdev->watchdog_timeo = NES_TX_TIMEOUT; - netdev->irq = nesdev->pcidev->irq; - netdev->max_mtu = NES_MAX_MTU; - netdev->hard_header_len = ETH_HLEN; - netdev->addr_len = ETH_ALEN; - netdev->type = ARPHRD_ETHER; - netdev->netdev_ops = &nes_netdev_ops; - netdev->ethtool_ops = &nes_ethtool_ops; - netif_napi_add(netdev, &nesvnic->napi, nes_netdev_poll, 128); - nes_debug(NES_DBG_INIT, "Enabling VLAN Insert/Delete.\n"); - - /* Fill in the port structure */ - nesvnic->netdev = netdev; - nesvnic->nesdev = nesdev; - nesvnic->msg_enable = netif_msg_init(debug, default_msg); - nesvnic->netdev_index = nesdev->netdev_count; - nesvnic->perfect_filter_index = nesdev->nesadapter->netdev_count; - nesvnic->max_frame_size = netdev->mtu + netdev->hard_header_len + VLAN_HLEN; - - curr_qp_map = nic_qp_mapping_per_function[PCI_FUNC(nesdev->pcidev->devfn)]; - nesvnic->nic.qp_id = curr_qp_map[nesdev->netdev_count].qpid; - nesvnic->nic_index = curr_qp_map[nesdev->netdev_count].nic_index; - nesvnic->logical_port = curr_qp_map[nesdev->netdev_count].logical_port; - - /* Setup the burned in MAC address */ - u64temp = (u64)nesdev->nesadapter->mac_addr_low; - u64temp += ((u64)nesdev->nesadapter->mac_addr_high) << 32; - u64temp += nesvnic->nic_index; - netdev->dev_addr[0] = (u8)(u64temp>>40); - netdev->dev_addr[1] = (u8)(u64temp>>32); - netdev->dev_addr[2] = (u8)(u64temp>>24); - netdev->dev_addr[3] = (u8)(u64temp>>16); - netdev->dev_addr[4] = (u8)(u64temp>>8); - netdev->dev_addr[5] = (u8)u64temp; - - netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX; - if ((nesvnic->logical_port < 2) || (nesdev->nesadapter->hw_rev != NE020_REV)) - netdev->hw_features |= NETIF_F_TSO; - - netdev->features = netdev->hw_features | NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX; - - nes_debug(NES_DBG_INIT, "nesvnic = %p, reported features = 0x%lX, QPid = %d," - " nic_index = %d, logical_port = %d, mac_index = %d.\n", - nesvnic, (unsigned long)netdev->features, nesvnic->nic.qp_id, - nesvnic->nic_index, nesvnic->logical_port, nesdev->mac_index); - - if (nesvnic->nesdev->nesadapter->port_count == 1 && - nesvnic->nesdev->nesadapter->adapter_fcn_count == 1) { - - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = nesvnic->nic_index + 1; - if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT) { - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } else { - nesvnic->qp_nic_index[2] = nesvnic->nic_index + 2; - nesvnic->qp_nic_index[3] = nesvnic->nic_index + 3; - } - } else { - if (nesvnic->nesdev->nesadapter->port_count == 2 || - (nesvnic->nesdev->nesadapter->port_count == 1 && - nesvnic->nesdev->nesadapter->adapter_fcn_count == 2)) { - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = nesvnic->nic_index - + 2; - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } else { - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = 0xf; - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } - } - nesvnic->next_qp_nic_index = 0; - - if (nesdev->netdev_count == 0) { - nesvnic->rdma_enabled = 1; - } else { - nesvnic->rdma_enabled = 0; - } - nesvnic->nic_cq.cq_number = nesvnic->nic.qp_id; - timer_setup(&nesvnic->event_timer, NULL, 0); - spin_lock_init(&nesvnic->tx_lock); - spin_lock_init(&nesvnic->port_ibevent_lock); - nesdev->netdev[nesdev->netdev_count] = netdev; - - nes_debug(NES_DBG_INIT, "Adding nesvnic (%p) to the adapters nesvnic_list for MAC%d.\n", - nesvnic, nesdev->mac_index); - list_add_tail(&nesvnic->list, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]); - - if ((nesdev->netdev_count == 0) && - ((PCI_FUNC(nesdev->pcidev->devfn) == nesdev->mac_index) || - ((phy_type == NES_PHY_TYPE_PUMA_1G) && - (((PCI_FUNC(nesdev->pcidev->devfn) == 1) && (nesdev->mac_index == 2)) || - ((PCI_FUNC(nesdev->pcidev->devfn) == 2) && (nesdev->mac_index == 1)))))) { - u32 u32temp; - u32 link_mask = 0; - u32 link_val = 0; - u16 temp_phy_data; - u16 phy_data = 0; - unsigned long flags; - - u32temp = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1))); - if (phy_type != NES_PHY_TYPE_PUMA_1G) { - u32temp |= 0x00200000; - nes_write_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1)), u32temp); - } - - /* Check and set linkup here. This is for back to back */ - /* configuration where second port won't get link interrupt */ - switch (phy_type) { - case NES_PHY_TYPE_PUMA_1G: - if (nesdev->mac_index < 2) { - link_mask = 0x01010000; - link_val = 0x01010000; - } else { - link_mask = 0x02020000; - link_val = 0x02020000; - } - break; - case NES_PHY_TYPE_SFP_D: - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - break; - default: - link_mask = 0x0f1f0000; - link_val = 0x0f0f0000; - break; - } - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1))); - - if (phy_type == NES_PHY_TYPE_SFP_D) { - if (phy_data & 0x0004) - nesvnic->linkup = 1; - } else { - if ((u32temp & link_mask) == link_val) - nesvnic->linkup = 1; - } - - /* clear the MAC interrupt status, assumes direct logical to physical mapping */ - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index)); - nes_debug(NES_DBG_INIT, "Phy interrupt status = 0x%X.\n", u32temp); - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index), u32temp); - - nes_init_phy(nesdev); - } - - nes_vlan_mode(netdev, nesdev, netdev->features); - - return netdev; -} - - -/** - * nes_netdev_destroy - destroy network device structure - */ -void nes_netdev_destroy(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - /* make sure 'stop' method is called by Linux stack */ - /* nes_netdev_stop(netdev); */ - - list_del(&nesvnic->list); - - if (nesvnic->of_device_registered) { - nes_destroy_ofa_device(nesvnic->nesibdev); - } - - free_netdev(netdev); -} - - -/** - * nes_nic_cm_xmit -- CM calls this to send out pkts - */ -int nes_nic_cm_xmit(struct sk_buff *skb, struct net_device *netdev) -{ - int ret; - - skb->dev = netdev; - ret = dev_queue_xmit(skb); - if (ret) { - nes_debug(NES_DBG_CM, "Bad return code from dev_queue_xmit %d\n", ret); - } - - return ret; -} diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c deleted file mode 100644 index e976292fc6c0..000000000000 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ /dev/null @@ -1,913 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "nes.h" - -static u16 nes_read16_eeprom(void __iomem *addr, u16 offset); - -u32 mh_detected; -u32 mh_pauses_sent; - -static u32 nes_set_pau(struct nes_device *nesdev) -{ - u32 ret = 0; - u32 counter; - - nes_write_indexed(nesdev, NES_IDX_GPR2, NES_ENABLE_PAU); - nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1); - - for (counter = 0; counter < NES_PAU_COUNTER; counter++) { - udelay(30); - if (!nes_read_indexed(nesdev, NES_IDX_GPR2)) { - printk(KERN_INFO PFX "PAU is supported.\n"); - break; - } - nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1); - } - if (counter == NES_PAU_COUNTER) { - printk(KERN_INFO PFX "PAU is not supported.\n"); - return -EPERM; - } - return ret; -} - -/** - * nes_read_eeprom_values - - */ -int nes_read_eeprom_values(struct nes_device *nesdev, struct nes_adapter *nesadapter) -{ - u32 mac_addr_low; - u16 mac_addr_high; - u16 eeprom_data; - u16 eeprom_offset; - u16 next_section_address; - u16 sw_section_ver; - u8 major_ver = 0; - u8 minor_ver = 0; - - /* TODO: deal with EEPROM endian issues */ - if (nesadapter->firmware_eeprom_offset == 0) { - /* Read the EEPROM Parameters */ - eeprom_data = nes_read16_eeprom(nesdev->regs, 0); - nes_debug(NES_DBG_HW, "EEPROM Offset 0 = 0x%04X\n", eeprom_data); - eeprom_offset = 2 + (((eeprom_data & 0x007f) << 3) << - ((eeprom_data & 0x0080) >> 7)); - nes_debug(NES_DBG_HW, "Firmware Offset = 0x%04X\n", eeprom_offset); - nesadapter->firmware_eeprom_offset = eeprom_offset; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4); - if (eeprom_data != 0x5746) { - nes_debug(NES_DBG_HW, "Not a valid Firmware Image = 0x%04X\n", eeprom_data); - return -1; - } - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - eeprom_offset += ((eeprom_data & 0x00ff) << 3) << ((eeprom_data & 0x0100) >> 8); - nes_debug(NES_DBG_HW, "Software Offset = 0x%04X\n", eeprom_offset); - nesadapter->software_eeprom_offset = eeprom_offset; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4); - if (eeprom_data != 0x5753) { - printk("Not a valid Software Image = 0x%04X\n", eeprom_data); - return -1; - } - sw_section_ver = nes_read16_eeprom(nesdev->regs, nesadapter->software_eeprom_offset + 6); - nes_debug(NES_DBG_HW, "Software section version number = 0x%04X\n", - sw_section_ver); - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) << - ((eeprom_data & 0x0100) >> 8)); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x414d) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) << - ((eeprom_data & 0x0100) >> 8)); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x4f52) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x4f52 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x5746) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5746 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x5753) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5753 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x414d) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x464e) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x464e but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 8); - printk(PFX "Firmware version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data); - major_ver = (u8)(eeprom_data >> 8); - minor_ver = (u8)(eeprom_data); - - if (nes_drv_opt & NES_DRV_OPT_DISABLE_VIRT_WQ) { - nes_debug(NES_DBG_HW, "Virtual WQs have been disabled\n"); - } else if (((major_ver == 2) && (minor_ver > 21)) || ((major_ver > 2) && (major_ver != 255))) { - nesadapter->virtwq = 1; - } - if (((major_ver == 3) && (minor_ver >= 16)) || (major_ver > 3)) - nesadapter->send_term_ok = 1; - - if (nes_drv_opt & NES_DRV_OPT_ENABLE_PAU) { - if (!nes_set_pau(nesdev)) - nesadapter->allow_unaligned_fpdus = 1; - } - - nesadapter->firmware_version = (((u32)(u8)(eeprom_data>>8)) << 16) + - (u32)((u8)eeprom_data); - - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 10); - printk(PFX "EEPROM version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data); - nesadapter->eeprom_version = (((u32)(u8)(eeprom_data>>8)) << 16) + - (u32)((u8)eeprom_data); - -no_fw_rev: - /* eeprom is valid */ - eeprom_offset = nesadapter->software_eeprom_offset; - eeprom_offset += 8; - nesadapter->netdev_max = (u8)nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_high = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_low = (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_low <<= 16; - mac_addr_low += (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "Base MAC Address = 0x%04X%08X\n", - mac_addr_high, mac_addr_low); - nes_debug(NES_DBG_HW, "MAC Address count = %u\n", nesadapter->netdev_max); - - nesadapter->mac_addr_low = mac_addr_low; - nesadapter->mac_addr_high = mac_addr_high; - - /* Read the Phy Type array */ - eeprom_offset += 10; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_type[0] = (u8)(eeprom_data >> 8); - nesadapter->phy_type[1] = (u8)eeprom_data; - - /* Read the port array */ - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_type[2] = (u8)(eeprom_data >> 8); - nesadapter->phy_type[3] = (u8)eeprom_data; - /* port_count is set by soft reset reg */ - nes_debug(NES_DBG_HW, "port_count = %u, port 0 -> %u, port 1 -> %u," - " port 2 -> %u, port 3 -> %u\n", - nesadapter->port_count, - nesadapter->phy_type[0], nesadapter->phy_type[1], - nesadapter->phy_type[2], nesadapter->phy_type[3]); - - /* Read PD config array */ - eeprom_offset += 10; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[0] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[0] = eeprom_data; - nes_debug(NES_DBG_HW, "PD0 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[0], nesadapter->pd_config_base[0]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[1] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[1] = eeprom_data; - nes_debug(NES_DBG_HW, "PD1 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[1], nesadapter->pd_config_base[1]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[2] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[2] = eeprom_data; - nes_debug(NES_DBG_HW, "PD2 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[2], nesadapter->pd_config_base[2]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[3] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[3] = eeprom_data; - nes_debug(NES_DBG_HW, "PD3 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[3], nesadapter->pd_config_base[3]); - - /* Read Rx Pool Size */ - eeprom_offset += 22; /* 46 */ - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->rx_pool_size = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "rx_pool_size = 0x%08X\n", nesadapter->rx_pool_size); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tx_pool_size = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tx_pool_size = 0x%08X\n", nesadapter->tx_pool_size); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->rx_threshold = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "rx_threshold = 0x%08X\n", nesadapter->rx_threshold); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tcp_timer_core_clk_divisor = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tcp_timer_core_clk_divisor = 0x%08X\n", - nesadapter->tcp_timer_core_clk_divisor); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->iwarp_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "iwarp_config = 0x%08X\n", nesadapter->iwarp_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->cm_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "cm_config = 0x%08X\n", nesadapter->cm_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->sws_timer_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "sws_timer_config = 0x%08X\n", nesadapter->sws_timer_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tcp_config1 = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tcp_config1 = 0x%08X\n", nesadapter->tcp_config1); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->wqm_wat = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "wqm_wat = 0x%08X\n", nesadapter->wqm_wat); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->core_clock = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "core_clock = 0x%08X\n", nesadapter->core_clock); - - if ((sw_section_ver) && (nesadapter->hw_rev != NE020_REV)) { - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_index[0] = (eeprom_data & 0xff00)>>8; - nesadapter->phy_index[1] = eeprom_data & 0x00ff; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_index[2] = (eeprom_data & 0xff00)>>8; - nesadapter->phy_index[3] = eeprom_data & 0x00ff; - } else { - nesadapter->phy_index[0] = 4; - nesadapter->phy_index[1] = 5; - nesadapter->phy_index[2] = 6; - nesadapter->phy_index[3] = 7; - } - nes_debug(NES_DBG_HW, "Phy address map = 0 > %u, 1 > %u, 2 > %u, 3 > %u\n", - nesadapter->phy_index[0],nesadapter->phy_index[1], - nesadapter->phy_index[2],nesadapter->phy_index[3]); - } - - return 0; -} - - -/** - * nes_read16_eeprom - */ -static u16 nes_read16_eeprom(void __iomem *addr, u16 offset) -{ - writel(NES_EEPROM_READ_REQUEST + (offset >> 1), - (void __iomem *)addr + NES_EEPROM_COMMAND); - - do { - } while (readl((void __iomem *)addr + NES_EEPROM_COMMAND) & - NES_EEPROM_READ_REQUEST); - - return readw((void __iomem *)addr + NES_EEPROM_DATA); -} - - -/** - * nes_write_1G_phy_reg - */ -void nes_write_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 data) -{ - u32 u32temp; - u32 counter; - - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x50020000 | data | ((u32)phy_reg << 18) | ((u32)phy_addr << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_read_1G_phy_reg - * This routine only issues the read, the data must be read - * separately. - */ -void nes_read_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 *data) -{ - u32 u32temp; - u32 counter; - - /* nes_debug(NES_DBG_PHY, "phy addr = %d, mac_index = %d\n", - phy_addr, nesdev->mac_index); */ - - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x60020000 | ((u32)phy_reg << 18) | ((u32)phy_addr << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) { - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - *data = 0xffff; - } else { - *data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - } -} - - -/** - * nes_write_10G_phy_reg - */ -void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_addr, u8 dev_addr, u16 phy_reg, - u16 data) -{ - u32 port_addr; - u32 u32temp; - u32 counter; - - port_addr = phy_addr; - - /* set address */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - - /* set data */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x10020000 | (u32)data | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_read_10G_phy_reg - * This routine only issues the read, the data must be read - * separately. - */ -void nes_read_10G_phy_reg(struct nes_device *nesdev, u8 phy_addr, u8 dev_addr, u16 phy_reg) -{ - u32 port_addr; - u32 u32temp; - u32 counter; - - port_addr = phy_addr; - - /* set address */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - - /* issue read */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x30020000 | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_get_cqp_request - */ -struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_cqp_request *cqp_request = NULL; - - if (!list_empty(&nesdev->cqp_avail_reqs)) { - spin_lock_irqsave(&nesdev->cqp.lock, flags); - if (!list_empty(&nesdev->cqp_avail_reqs)) { - cqp_request = list_entry(nesdev->cqp_avail_reqs.next, - struct nes_cqp_request, list); - list_del_init(&cqp_request->list); - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - } - if (cqp_request == NULL) { - cqp_request = kzalloc(sizeof(struct nes_cqp_request), GFP_ATOMIC); - if (cqp_request) { - cqp_request->dynamic = 1; - INIT_LIST_HEAD(&cqp_request->list); - } - } - - if (cqp_request) { - init_waitqueue_head(&cqp_request->waitq); - cqp_request->waiting = 0; - cqp_request->request_done = 0; - cqp_request->callback = 0; - nes_debug(NES_DBG_CQP, "Got cqp request %p from the available list \n", - cqp_request); - } - - return cqp_request; -} - -void nes_free_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - unsigned long flags; - - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n", - cqp_request, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX]) & 0x3f); - - if (cqp_request->dynamic) { - kfree(cqp_request); - } else { - spin_lock_irqsave(&nesdev->cqp.lock, flags); - list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - } -} - -void nes_put_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - if (atomic_dec_and_test(&cqp_request->refcount)) - nes_free_cqp_request(nesdev, cqp_request); -} - - -/** - * nes_post_cqp_request - */ -void nes_post_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - unsigned long flags; - u32 cqp_head; - u64 u64temp; - u32 opcode; - int ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX; - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - if (((((nesdev->cqp.sq_tail+(nesdev->cqp.sq_size*2))-nesdev->cqp.sq_head) & - (nesdev->cqp.sq_size - 1)) != 1) - && (list_empty(&nesdev->cqp_pending_reqs))) { - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe)); - opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]); - if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT) - ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX; - barrier(); - u64temp = (unsigned long)cqp_request; - set_wqe_64bit_value(cqp_wqe->wqe_words, ctx_index, u64temp); - nes_debug(NES_DBG_CQP, "CQP request (opcode 0x%02X), line 1 = 0x%08X put on CQPs SQ," - " request = %p, cqp_head = %u, cqp_tail = %u, cqp_size = %u," - " waiting = %d, refcount = %d.\n", - opcode & NES_CQP_OPCODE_MASK, - le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX]), cqp_request, - nesdev->cqp.sq_head, nesdev->cqp.sq_tail, nesdev->cqp.sq_size, - cqp_request->waiting, atomic_read(&cqp_request->refcount)); - - barrier(); - - /* Ring doorbell (1 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id); - - barrier(); - } else { - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X), line 1 = 0x%08X" - " put on the pending queue.\n", - cqp_request, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_ID_IDX])); - list_add_tail(&cqp_request->list, &nesdev->cqp_pending_reqs); - } - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - return; -} - -/** - * nes_arp_table - */ -int nes_arp_table(struct nes_device *nesdev, u32 ip_addr, u8 *mac_addr, u32 action) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - int arp_index; - int err = 0; - __be32 tmp_addr; - - for (arp_index = 0; (u32) arp_index < nesadapter->arp_table_size; arp_index++) { - if (nesadapter->arp_table[arp_index].ip_addr == ip_addr) - break; - } - - if (action == NES_ARP_ADD) { - if (arp_index != nesadapter->arp_table_size) { - return -1; - } - - arp_index = 0; - err = nes_alloc_resource(nesadapter, nesadapter->allocated_arps, - nesadapter->arp_table_size, (u32 *)&arp_index, &nesadapter->next_arp_index, NES_RESOURCE_ARP); - if (err) { - nes_debug(NES_DBG_NETDEV, "nes_alloc_resource returned error = %u\n", err); - return err; - } - nes_debug(NES_DBG_NETDEV, "ADD, arp_index=%d\n", arp_index); - - nesadapter->arp_table[arp_index].ip_addr = ip_addr; - memcpy(nesadapter->arp_table[arp_index].mac_addr, mac_addr, ETH_ALEN); - return arp_index; - } - - /* DELETE or RESOLVE */ - if (arp_index == nesadapter->arp_table_size) { - tmp_addr = cpu_to_be32(ip_addr); - nes_debug(NES_DBG_NETDEV, "MAC for %pI4 not in ARP table - cannot %s\n", - &tmp_addr, action == NES_ARP_RESOLVE ? "resolve" : "delete"); - return -1; - } - - if (action == NES_ARP_RESOLVE) { - nes_debug(NES_DBG_NETDEV, "RESOLVE, arp_index=%d\n", arp_index); - return arp_index; - } - - if (action == NES_ARP_DELETE) { - nes_debug(NES_DBG_NETDEV, "DELETE, arp_index=%d\n", arp_index); - nesadapter->arp_table[arp_index].ip_addr = 0; - eth_zero_addr(nesadapter->arp_table[arp_index].mac_addr); - nes_free_resource(nesadapter, nesadapter->allocated_arps, arp_index); - return arp_index; - } - - return -1; -} - - -/** - * nes_mh_fix - */ -void nes_mh_fix(struct timer_list *t) -{ - struct nes_adapter *nesadapter = from_timer(nesadapter, t, mh_timer); - struct nes_device *nesdev = nesadapter->nesdev; - unsigned long flags; - struct nes_vnic *nesvnic; - u32 used_chunks_tx; - u32 temp_used_chunks_tx; - u32 temp_last_used_chunks_tx; - u32 used_chunks_mask; - u32 mac_tx_frames_low; - u32 mac_tx_frames_high; - u32 mac_tx_pauses; - u32 reset_value; - u32 tx_control; - u32 tx_config; - u32 tx_pause_quanta; - u32 rx_control; - u32 rx_config; - u32 mac_exact_match; - u32 mpp_debug; - u32 i=0; - u32 chunks_tx_progress = 0; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - if ((nesadapter->mac_sw_state[0] != NES_MAC_SW_IDLE) || (nesadapter->mac_link_down[0])) { - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - goto no_mh_work; - } - nesadapter->mac_sw_state[0] = NES_MAC_SW_MH; - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - do { - mac_tx_frames_low = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_LOW); - mac_tx_frames_high = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_HIGH); - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - used_chunks_tx = nes_read_indexed(nesdev, NES_IDX_USED_CHUNKS_TX); - nesdev->mac_pause_frames_sent += mac_tx_pauses; - used_chunks_mask = 0; - temp_used_chunks_tx = used_chunks_tx; - temp_last_used_chunks_tx = nesdev->last_used_chunks_tx; - - if (nesdev->netdev[0]) { - nesvnic = netdev_priv(nesdev->netdev[0]); - } else { - break; - } - - for (i=0; i<4; i++) { - used_chunks_mask <<= 8; - if (nesvnic->qp_nic_index[i] != 0xff) { - used_chunks_mask |= 0xff; - if ((temp_used_chunks_tx&0xff)<(temp_last_used_chunks_tx&0xff)) { - chunks_tx_progress = 1; - } - } - temp_used_chunks_tx >>= 8; - temp_last_used_chunks_tx >>= 8; - } - if ((mac_tx_frames_low) || (mac_tx_frames_high) || - (!(used_chunks_tx&used_chunks_mask)) || - (!(nesdev->last_used_chunks_tx&used_chunks_mask)) || - (chunks_tx_progress) ) { - nesdev->last_used_chunks_tx = used_chunks_tx; - break; - } - nesdev->last_used_chunks_tx = used_chunks_tx; - barrier(); - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000005); - mh_pauses_sent++; - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - if (mac_tx_pauses) { - nesdev->mac_pause_frames_sent += mac_tx_pauses; - break; - } - - tx_control = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONTROL); - tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG); - tx_pause_quanta = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA); - rx_control = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONTROL); - rx_config = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONFIG); - mac_exact_match = nes_read_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM); - mpp_debug = nes_read_indexed(nesdev, NES_IDX_MPP_DEBUG); - - /* one last ditch effort to avoid a false positive */ - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - if (mac_tx_pauses) { - nesdev->last_mac_tx_pauses = nesdev->mac_pause_frames_sent; - nes_debug(NES_DBG_HW, "failsafe caught slow outbound pause\n"); - break; - } - mh_detected++; - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, 0x00000000); - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value | 0x0000001d); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)) { - /* mdelay(1); */ - } - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008); - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0); - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000); - if (nesadapter->OneG_Mode) { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222); - } else { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222); - } - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_STATUS0); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff); - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, tx_control); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA, tx_pause_quanta); - nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONTROL, rx_control); - nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONFIG, rx_config); - nes_write_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM, mac_exact_match); - nes_write_indexed(nesdev, NES_IDX_MPP_DEBUG, mpp_debug); - - } while (0); - - nesadapter->mac_sw_state[0] = NES_MAC_SW_IDLE; -no_mh_work: - nesdev->nesadapter->mh_timer.expires = jiffies + (HZ/5); - add_timer(&nesdev->nesadapter->mh_timer); -} - -/** - * nes_clc - */ -void nes_clc(struct timer_list *t) -{ - struct nes_adapter *nesadapter = from_timer(nesadapter, t, lc_timer); - unsigned long flags; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nesadapter->link_interrupt_count[0] = 0; - nesadapter->link_interrupt_count[1] = 0; - nesadapter->link_interrupt_count[2] = 0; - nesadapter->link_interrupt_count[3] = 0; - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */ - add_timer(&nesadapter->lc_timer); -} - - -/** - * nes_dump_mem - */ -void nes_dump_mem(unsigned int dump_debug_level, void *addr, int length) -{ - if (!(nes_debug_level & dump_debug_level)) { - return; - } - - if (length > 0x100) { - nes_debug(dump_debug_level, "Length truncated from %x to %x\n", length, 0x100); - length = 0x100; - } - nes_debug(dump_debug_level, "Address=0x%p, length=0x%x (%d)\n", addr, length, length); - - print_hex_dump(KERN_ERR, PFX, DUMP_PREFIX_NONE, 16, 1, addr, length, true); -} diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c deleted file mode 100644 index 0420203820f6..000000000000 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ /dev/null @@ -1,3721 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "nes.h" - -#include - -atomic_t mod_qp_timouts; -atomic_t qps_created; -atomic_t sw_qps_destroyed; - -static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev); -static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); - -/** - * nes_alloc_mw - */ -static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type, - struct ib_udata *udata) -{ - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_cqp_request *cqp_request; - struct nes_mr *nesmr; - struct ib_mw *ibmw; - struct nes_hw_cqp_wqe *cqp_wqe; - int ret; - u32 stag; - u32 stag_index = 0; - u32 next_stag_index = 0; - u32 driver_key = 0; - u8 stag_key = 0; - - if (type != IB_MW_TYPE_1) - return ERR_PTR(-EINVAL); - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = 0; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW); - if (ret) { - return ERR_PTR(ret); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", - stag, stag_index); - - /* Register the region with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = - cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ | - NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_REM_ACC_EN); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - stag, ret, cqp_request->major_code, cqp_request->minor_code); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - if (!ret) { - return ERR_PTR(-ETIME); - } else { - return ERR_PTR(-ENOMEM); - } - } - nes_put_cqp_request(nesdev, cqp_request); - - nesmr->ibmw.rkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MW; - ibmw = &nesmr->ibmw; - nesmr->pbl_4k = 0; - nesmr->pbls_used = 0; - - return ibmw; -} - - -/** - * nes_dealloc_mw - */ -static int nes_dealloc_mw(struct ib_mw *ibmw) -{ - struct nes_mr *nesmr = to_nesmw(ibmw); - struct nes_vnic *nesvnic = to_nesvnic(ibmw->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - int err = 0; - int ret; - - /* Deallocate the window with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n", - ibmw->rkey); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - ret, cqp_request->major_code, cqp_request->minor_code); - if (!ret) - err = -ETIME; - else if (cqp_request->major_code) - err = -EIO; - - nes_put_cqp_request(nesdev, cqp_request); - - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - (ibmw->rkey & 0x0fffff00) >> 8); - kfree(nesmr); - - return err; -} - - -/* - * nes_alloc_fast_mr - */ -static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, - u32 stag, u32 page_count) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 opcode = 0; - u16 major_code; - u64 region_length = page_count * PAGE_SIZE; - - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, " - "region_length = %llu\n", - page_count, region_length); - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nesadapter->free_4kpbl > 0) { - nesadapter->free_4kpbl--; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } else { - /* No 4kpbl's available: */ - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_debug(NES_DBG_MR, "Out of Pbls\n"); - nes_free_cqp_request(nesdev, cqp_request); - return -ENOMEM; - } - - opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR | - NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_REM_ACC_EN; - /* - * The current OFED API does not support the zero based TO option. - * If added then need to changed the NES_CQP_STAG_VA* option. Also, - * the API does not support that ability to have the MR set for local - * access only when created and not allow the SQ op to override. Given - * this the remote enable must be set here. - */ - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1); - - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = - cpu_to_le32((u32)(region_length >> 8) & 0xff000000); - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= - cpu_to_le32(nespd->pd_id & 0x00007fff); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8)); - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); - barrier(); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, - (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, " - "wait_event_timeout ret = %u, CQP Major:Minor codes = " - "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code, - cqp_request->minor_code); - major_code = cqp_request->major_code; - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret || major_code) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_4kpbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - return 0; -} - -/* - * nes_alloc_mr - */ -static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata) -{ - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - u32 next_stag_index; - u8 stag_key = 0; - u32 driver_key = 0; - int err = 0; - u32 stag_index = 0; - struct nes_mr *nesmr; - u32 stag; - int ret; - struct ib_mr *ibmr; - - if (mr_type != IB_MR_TYPE_MEM_REG) - return ERR_PTR(-EINVAL); - - if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) - return ERR_PTR(-E2BIG); - -/* - * Note: Set to always use a fixed length single page entry PBL. This is to allow - * for the fast_reg_mr operation to always know the size of the PBL. - */ - if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) - return ERR_PTR(-E2BIG); - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, - &next_stag_index, NES_RESOURCE_FAST_MR); - if (err) - return ERR_PTR(err); - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n", - stag, stag_index); - - ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_FMEM; - ibmr = &nesmr->ibmr; - } else { - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - nesmr->pages = pci_alloc_consistent(nesdev->pcidev, - max_num_sg * sizeof(u64), - &nesmr->paddr); - if (!nesmr->paddr) - goto err; - - nesmr->max_pages = max_num_sg; - - return ibmr; - -err: - nes_dereg_mr(ibmr, udata); - - return ERR_PTR(-ENOMEM); -} - -static int nes_set_page(struct ib_mr *ibmr, u64 addr) -{ - struct nes_mr *nesmr = to_nesmr(ibmr); - - if (unlikely(nesmr->npages == nesmr->max_pages)) - return -ENOMEM; - - nesmr->pages[nesmr->npages++] = cpu_to_le64(addr); - - return 0; -} - -static int nes_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, - int sg_nents, unsigned int *sg_offset) -{ - struct nes_mr *nesmr = to_nesmr(ibmr); - - nesmr->npages = 0; - - return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, nes_set_page); -} - -/** - * nes_query_device - */ -static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props, - struct ib_udata *uhw) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - - if (uhw->inlen || uhw->outlen) - return -EINVAL; - - memset(props, 0, sizeof(*props)); - memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6); - - props->fw_ver = nesdev->nesadapter->firmware_version; - props->device_cap_flags = nesdev->nesadapter->device_cap_flags; - props->vendor_id = nesdev->nesadapter->vendor_id; - props->vendor_part_id = nesdev->nesadapter->vendor_part_id; - props->hw_ver = nesdev->nesadapter->hw_rev; - props->max_mr_size = 0x80000000; - props->max_qp = nesibdev->max_qp; - props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2; - props->max_send_sge = nesdev->nesadapter->max_sge; - props->max_recv_sge = nesdev->nesadapter->max_sge; - props->max_cq = nesibdev->max_cq; - props->max_cqe = nesdev->nesadapter->max_cqe; - props->max_mr = nesibdev->max_mr; - props->max_mw = nesibdev->max_mr; - props->max_pd = nesibdev->max_pd; - props->max_sge_rd = 1; - switch (nesdev->nesadapter->max_irrq_wr) { - case 0: - props->max_qp_rd_atom = 2; - break; - case 1: - props->max_qp_rd_atom = 8; - break; - case 2: - props->max_qp_rd_atom = 32; - break; - case 3: - props->max_qp_rd_atom = 64; - break; - default: - props->max_qp_rd_atom = 0; - } - props->max_qp_init_rd_atom = props->max_qp_rd_atom; - props->atomic_cap = IB_ATOMIC_NONE; - props->max_map_per_fmr = 1; - - return 0; -} - - -/** - * nes_query_port - */ -static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct net_device *netdev = nesvnic->netdev; - - /* props being zeroed by the caller, avoid zeroing it here */ - - props->max_mtu = IB_MTU_4096; - props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); - - props->lid = 1; - if (netif_queue_stopped(netdev)) - props->state = IB_PORT_DOWN; - else if (nesvnic->linkup) - props->state = IB_PORT_ACTIVE; - else - props->state = IB_PORT_DOWN; - props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | - IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; - props->gid_tbl_len = 1; - props->pkey_tbl_len = 1; - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_SDR; - props->max_msg_sz = 0x80000000; - - return 0; -} - -/** - * nes_query_pkey - */ -static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) -{ - *pkey = 0; - return 0; -} - - -/** - * nes_query_gid - */ -static int nes_query_gid(struct ib_device *ibdev, u8 port, - int index, union ib_gid *gid) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - - memset(&(gid->raw[0]), 0, sizeof(gid->raw)); - memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6); - - return 0; -} - - -/** - * nes_alloc_ucontext - Allocate the user context data structure. This keeps track - * of all objects associated with a particular user-mode client. - */ -static int nes_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) -{ - struct ib_device *ibdev = uctx->device; - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_alloc_ucontext_req req; - struct nes_alloc_ucontext_resp uresp = {}; - struct nes_ucontext *nes_ucontext = to_nesucontext(uctx); - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - - - if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) { - printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n"); - return -EINVAL; - } - - if (req.userspace_ver != NES_ABI_USERSPACE_VER) { - printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n", - req.userspace_ver, NES_ABI_USERSPACE_VER); - return -EINVAL; - } - - - uresp.max_qps = nesibdev->max_qp; - uresp.max_pds = nesibdev->max_pd; - uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2; - uresp.virtwq = nesadapter->virtwq; - uresp.kernel_ver = NES_ABI_KERNEL_VER; - - nes_ucontext->nesdev = nesdev; - nes_ucontext->mmap_wq_offset = uresp.max_pds; - nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset + - ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) / - PAGE_SIZE; - - - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) - return -EFAULT; - - INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list); - INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list); - return 0; -} - -/** - * nes_dealloc_ucontext - */ -static void nes_dealloc_ucontext(struct ib_ucontext *context) -{ - return; -} - -/** - * nes_mmap - */ -static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) -{ - unsigned long index; - struct nes_vnic *nesvnic = to_nesvnic(context->device); - struct nes_device *nesdev = nesvnic->nesdev; - /* struct nes_adapter *nesadapter = nesdev->nesadapter; */ - struct nes_ucontext *nes_ucontext; - struct nes_qp *nesqp; - - nes_ucontext = to_nesucontext(context); - - - if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) { - index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE; - index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) + - PAGE_SIZE-1) & (~(PAGE_SIZE-1)); - if (!test_bit(index, nes_ucontext->allocated_wqs)) { - nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index); - return -EFAULT; - } - nesqp = nes_ucontext->mmap_nesqp[index]; - if (nesqp == NULL) { - nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index); - return -EFAULT; - } - if (remap_pfn_range(vma, vma->vm_start, - virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT, - vma->vm_end - vma->vm_start, - vma->vm_page_prot)) { - nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n"); - return -EAGAIN; - } - vma->vm_private_data = nesqp; - return 0; - } else { - index = vma->vm_pgoff; - if (!test_bit(index, nes_ucontext->allocated_doorbells)) - return -EFAULT; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - if (io_remap_pfn_range(vma, vma->vm_start, - (nesdev->doorbell_start + - ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096)) - >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - vma->vm_private_data = nes_ucontext; - return 0; - } - - return -ENOSYS; -} - - -/** - * nes_alloc_pd - */ -static int nes_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) -{ - struct ib_device *ibdev = pd->device; - struct nes_pd *nespd = to_nespd(pd); - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_alloc_pd_resp uresp; - u32 pd_num = 0; - int err; - struct nes_ucontext *nesucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - - nes_debug( - NES_DBG_PD, - "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n", - nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, - &nesucontext->ibucontext, netdev_refcnt_read(nesvnic->netdev)); - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds, - nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD); - if (err) - return err; - - nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n", - nespd, dev_name(&nesvnic->nesibdev->ibdev.dev)); - - nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd; - - if (udata) { - nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells, - NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db); - nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n", - nespd->mmap_db_index, nespd->pd_id); - if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) { - nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n"); - nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); - return -ENOMEM; - } - - uresp.pd_id = nespd->pd_id; - uresp.mmap_db_index = nespd->mmap_db_index; - if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) { - nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); - return -EFAULT; - } - - set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); - nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id; - nesucontext->first_free_db = nespd->mmap_db_index + 1; - } - - nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd); - return 0; -} - - -/** - * nes_dealloc_pd - */ -static void nes_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) -{ - struct nes_ucontext *nesucontext; - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if (udata) { - nesucontext = - rdma_udata_to_drv_context( - udata, - struct nes_ucontext, - ibucontext); - nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n", - nespd->mmap_db_index); - clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); - nesucontext->mmap_db_index[nespd->mmap_db_index] = 0; - if (nesucontext->first_free_db > nespd->mmap_db_index) { - nesucontext->first_free_db = nespd->mmap_db_index; - } - } - - nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n", - nespd->pd_id, nespd); - nes_free_resource(nesadapter, nesadapter->allocated_pds, - (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12)); -} - - -/** - * nes_get_encoded_size - */ -static inline u8 nes_get_encoded_size(int *size) -{ - u8 encoded_size = 0; - if (*size <= 32) { - *size = 32; - encoded_size = 1; - } else if (*size <= 128) { - *size = 128; - encoded_size = 2; - } else if (*size <= 512) { - *size = 512; - encoded_size = 3; - } - return (encoded_size); -} - - - -/** - * nes_setup_virt_qp - */ -static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl, - struct nes_vnic *nesvnic, int sq_size, int rq_size) -{ - unsigned long flags; - void *mem; - __le64 *pbl = NULL; - __le64 *tpbl; - __le64 *pblbuffer; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 pbl_entries; - u8 rq_pbl_entries; - u8 sq_pbl_entries; - - pbl_entries = nespbl->pbl_size >> 3; - nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n", - nespbl->pbl_size, pbl_entries, - (void *)nespbl->pbl_vbase, - (unsigned long) nespbl->pbl_pbase); - pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */ - /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */ - /* the first pbl to be fro the rq_vbase... */ - rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; - sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; - nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); - if (!nespbl->page) { - nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n"); - kfree(nespbl); - return -ENOMEM; - } - - nesqp->hwqp.sq_vbase = kmap(nespbl->page); - nesqp->page = nespbl->page; - if (!nesqp->hwqp.sq_vbase) { - nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n"); - kfree(nespbl); - return -ENOMEM; - } - - /* Now to get to sq.. we need to calculate how many */ - /* PBL entries were used by the rq.. */ - pbl += sq_pbl_entries; - nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); - /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */ - /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */ - - nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n", - nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase, - nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (!nesadapter->free_256pbl) { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - kfree(nespbl); - return -ENOMEM; - } - nesadapter->free_256pbl--; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase); - pblbuffer = nesqp->pbl_vbase; - if (!nesqp->pbl_vbase) { - /* memory allocated during nes_reg_user_mr() */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - return -ENOMEM; - } - memset(nesqp->pbl_vbase, 0, 256); - /* fill in the page address in the pbl buffer.. */ - tpbl = pblbuffer + 16; - pbl = (__le64 *)nespbl->pbl_vbase; - while (sq_pbl_entries--) - *tpbl++ = *pbl++; - tpbl = pblbuffer; - while (rq_pbl_entries--) - *tpbl++ = *pbl++; - - /* done with memory allocated during nes_reg_user_mr() */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - - nesqp->qp_mem_size = - max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */ - /* Round up to a multiple of a page */ - nesqp->qp_mem_size += PAGE_SIZE - 1; - nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); - - mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, - &nesqp->hwqp.q2_pbase); - - if (!mem) { - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); - nesqp->pbl_vbase = NULL; - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - return -ENOMEM; - } - nesqp->sq_kmapped = 1; - nesqp->hwqp.q2_vbase = mem; - mem += 256; - memset(nesqp->hwqp.q2_vbase, 0, 256); - nesqp->nesqp_context = mem; - memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); - nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; - - return 0; -} - - -/** - * nes_setup_mmap_qp - */ -static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic, - int sq_size, int rq_size) -{ - void *mem; - struct nes_device *nesdev = nesvnic->nesdev; - - nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) + - (sizeof(struct nes_hw_qp_wqe) * rq_size) + - max((u32)sizeof(struct nes_qp_context), ((u32)256)) + - 256; /* this is Q2 */ - /* Round up to a multiple of a page */ - nesqp->qp_mem_size += PAGE_SIZE - 1; - nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); - - mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, - &nesqp->hwqp.sq_pbase); - if (!mem) - return -ENOMEM; - nes_debug(NES_DBG_QP, "PCI consistent memory for " - "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n", - mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size); - - memset(mem, 0, nesqp->qp_mem_size); - - nesqp->hwqp.sq_vbase = mem; - mem += sizeof(struct nes_hw_qp_wqe) * sq_size; - - nesqp->hwqp.rq_vbase = mem; - nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase + - sizeof(struct nes_hw_qp_wqe) * sq_size; - mem += sizeof(struct nes_hw_qp_wqe) * rq_size; - - nesqp->hwqp.q2_vbase = mem; - nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase + - sizeof(struct nes_hw_qp_wqe) * rq_size; - mem += 256; - memset(nesqp->hwqp.q2_vbase, 0, 256); - - nesqp->nesqp_context = mem; - nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; - memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); - return 0; -} - - -/** - * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory. - */ -static void nes_free_qp_mem(struct nes_device *nesdev, - struct nes_qp *nesqp, int virt_wqs) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - if (!virt_wqs) { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); - }else { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase ); - nesqp->pbl_vbase = NULL; - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - } -} - - -/** - * nes_create_qp - */ -static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, struct ib_udata *udata) -{ - u64 u64temp= 0; - u64 u64nesqp = 0; - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_qp *nesqp; - struct nes_cq *nescq; - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - struct nes_create_qp_req req; - struct nes_create_qp_resp uresp; - struct nes_pbl *nespbl = NULL; - u32 qp_num = 0; - u32 opcode = 0; - /* u32 counter = 0; */ - void *mem; - unsigned long flags; - int ret; - int err; - int virt_wqs = 0; - int sq_size; - int rq_size; - u8 sq_encoded_size; - u8 rq_encoded_size; - /* int counter; */ - - if (init_attr->create_flags) - return ERR_PTR(-EINVAL); - - atomic_inc(&qps_created); - switch (init_attr->qp_type) { - case IB_QPT_RC: - if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) { - init_attr->cap.max_inline_data = 0; - } else { - init_attr->cap.max_inline_data = 64; - } - sq_size = init_attr->cap.max_send_wr; - rq_size = init_attr->cap.max_recv_wr; - - /* check if the encoded sizes are OK or not... */ - sq_encoded_size = nes_get_encoded_size(&sq_size); - rq_encoded_size = nes_get_encoded_size(&rq_size); - - if ((!sq_encoded_size) || (!rq_encoded_size)) { - nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n", - rq_size, sq_size); - return ERR_PTR(-EINVAL); - } - - init_attr->cap.max_send_wr = sq_size -2; - init_attr->cap.max_recv_wr = rq_size -1; - nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size); - - ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps, - nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP); - if (ret) { - return ERR_PTR(ret); - } - - /* Need 512 (actually now 1024) byte alignment on this structure */ - mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL); - if (!mem) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - return ERR_PTR(-ENOMEM); - } - u64nesqp = (unsigned long)mem; - u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1; - u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1; - u64nesqp &= ~u64temp; - nesqp = (struct nes_qp *)(unsigned long)u64nesqp; - /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n", - nesqp, mem, NES_SW_CONTEXT_ALIGN); */ - nesqp->allocated_buffer = mem; - - if (udata) { - if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n"); - return ERR_PTR(-EFAULT); - } - if (req.user_wqe_buffers) { - virt_wqs = 1; - } - if (req.user_qp_buffer) - nesqp->nesuqp_addr = req.user_qp_buffer; - - nesqp->user_mode = 1; - if (virt_wqs) { - err = 1; - list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) { - if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) { - list_del(&nespbl->list); - err = 0; - nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n", - nespbl, nespbl->user_base); - break; - } - } - if (err) { - nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n", - (long long unsigned int)req.user_wqe_buffers); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-EFAULT); - } - } - - nesqp->mmap_sq_db_index = - find_next_zero_bit(nes_ucontext->allocated_wqs, - NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq); - /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n", - nespd->mmap_db_index); */ - if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) { - nes_debug(NES_DBG_QP, - "db index > max user regions, failing create QP\n"); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - if (virt_wqs) { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - } - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); - nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp; - nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1; - } - err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) : - nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size); - if (err) { - nes_debug(NES_DBG_QP, - "error geting qp mem code = %d\n", err); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - - nesqp->hwqp.sq_size = sq_size; - nesqp->hwqp.sq_encoded_size = sq_encoded_size; - nesqp->hwqp.sq_head = 1; - nesqp->hwqp.rq_size = rq_size; - nesqp->hwqp.rq_encoded_size = rq_encoded_size; - /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n", - (void *)nesqp->nesqp_context_pbase); - */ - nesqp->hwqp.qp_id = qp_num; - nesqp->ibqp.qp_num = nesqp->hwqp.qp_id; - nesqp->nespd = nespd; - - nescq = to_nescq(init_attr->send_cq); - nesqp->nesscq = nescq; - nescq = to_nescq(init_attr->recv_cq); - nesqp->nesrcq = nescq; - - nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC_PCI_FCN_SHIFT); - nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size << - NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT); - nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size << - NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT); - if (!udata) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN); - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN); - } - nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number + - ((u32)nesqp->nesrcq->hw_cq.cq_number << 16)); - u64temp = (u64)nesqp->hwqp.sq_pbase; - nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - - - if (!virt_wqs) { - u64temp = (u64)nesqp->hwqp.sq_pbase; - nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)nesqp->hwqp.rq_pbase; - nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - } else { - u64temp = (u64)nesqp->pbl_pbase; - nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - } - - /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n", - nesvnic->next_qp_nic_index, - nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] << - NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT); - nesvnic->next_qp_nic_index++; - if ((nesvnic->next_qp_nic_index > 3) || - (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) { - nesvnic->next_qp_nic_index = 0; - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16); - u64temp = (u64)nesqp->hwqp.q2_pbase; - nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp))); - nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp)))); - nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM | - NES_QPCONTEXT_ORDIRD_AAH | - ((((u32)nesadapter->max_irrq_wr) << - NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK)); - if (disable_mpa_crc) { - nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n"); - nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC); - } - - - /* Create the QP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n"); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - if (!virt_wqs) { - opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | - NES_CQP_QP_IWARP_STATE_IDLE; - } else { - opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS | - NES_CQP_QP_IWARP_STATE_IDLE; - } - opcode |= NES_CQP_QP_CQS_VALID; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n", - nesqp->hwqp.qp_id); - ret = wait_event_timeout(cqp_request->waitq, - (cqp_request->request_done != 0), NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u," - " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail, - cqp_request->major_code, cqp_request->minor_code); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - if (!ret) { - return ERR_PTR(-ETIME); - } else { - return ERR_PTR(-EIO); - } - } - - nes_put_cqp_request(nesdev, cqp_request); - - if (udata) { - uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index; - uresp.mmap_rq_db_index = 0; - uresp.actual_sq_size = sq_size; - uresp.actual_rq_size = rq_size; - uresp.qp_id = nesqp->hwqp.qp_id; - uresp.nes_drv_opt = nes_drv_opt; - if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-EFAULT); - } - } - - nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n", - nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp)); - spin_lock_init(&nesqp->lock); - nes_add_ref(&nesqp->ibqp); - break; - default: - nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type); - return ERR_PTR(-EINVAL); - } - init_completion(&nesqp->sq_drained); - init_completion(&nesqp->rq_drained); - - nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); - timer_setup(&nesqp->terminate_timer, nes_terminate_timeout, 0); - - /* update the QP table */ - nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp; - nes_debug(NES_DBG_QP, "netdev refcnt=%u\n", - netdev_refcnt_read(nesvnic->netdev)); - - return &nesqp->ibqp; -} - -/** - * nes_clean_cq - */ -static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq) -{ - u32 cq_head; - u32 lo; - u32 hi; - u64 u64temp; - unsigned long flags = 0; - - spin_lock_irqsave(&nescq->lock, flags); - - cq_head = nescq->hw_cq.cq_head; - while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) { - rmb(); - lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]); - u64temp = (((u64)hi) << 32) | ((u64)lo); - u64temp &= ~(NES_SW_CONTEXT_ALIGN-1); - if (u64temp == (u64)(unsigned long)nesqp) { - /* Zero the context value so cqe will be ignored */ - nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0; - nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0; - } - - if (++cq_head >= nescq->hw_cq.cq_size) - cq_head = 0; - } - - spin_unlock_irqrestore(&nescq->lock, flags); -} - - -/** - * nes_destroy_qp - */ -static int nes_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_ucontext *nes_ucontext; - struct ib_qp_attr attr; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - int ret = 0; - - atomic_inc(&sw_qps_destroyed); - nesqp->destroyed = 1; - - /* Blow away the connection if it exists. */ - if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) { - /* if (nesqp->ibqp_state == IB_QPS_RTS) { */ - attr.qp_state = IB_QPS_ERR; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - } - - if (((nesqp->ibqp_state == IB_QPS_INIT) || - (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) { - cm_id = nesqp->cm_id; - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ETIMEDOUT; - cm_event.local_addr = cm_id->local_addr; - cm_event.remote_addr = cm_id->remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for " - "QP%u. cm_id = %p, refcount = %u. \n", - nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount)); - - cm_id->rem_ref(cm_id); - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret); - } - - if (nesqp->user_mode) { - if (udata) { - nes_ucontext = - rdma_udata_to_drv_context( - udata, - struct nes_ucontext, - ibucontext); - clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); - nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL; - if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) { - nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index; - } - } - if (nesqp->pbl_pbase && nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - } else { - /* Clean any pending completions from the cq(s) */ - if (nesqp->nesscq) - nes_clean_cq(nesqp, nesqp->nesscq); - - if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq)) - nes_clean_cq(nesqp, nesqp->nesrcq); - } - nes_rem_ref(&nesqp->ibqp); - return 0; -} - - -/** - * nes_create_cq - */ -static int nes_create_cq(struct ib_cq *ibcq, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -{ - struct ib_device *ibdev = ibcq->device; - int entries = attr->cqe; - u64 u64temp; - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_cq *nescq = to_nescq(ibcq); - struct nes_ucontext *nes_ucontext = NULL; - struct nes_cqp_request *cqp_request; - void *mem = NULL; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_pbl *nespbl = NULL; - struct nes_create_cq_req req; - struct nes_create_cq_resp resp; - u32 cq_num = 0; - u32 opcode = 0; - u32 pbl_entries = 1; - int err; - unsigned long flags; - int ret; - - if (attr->flags) - return -EINVAL; - - if (entries > nesadapter->max_cqe) - return -EINVAL; - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, - nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ); - if (err) - return err; - - nescq->hw_cq.cq_size = max(entries + 1, 5); - nescq->hw_cq.cq_number = cq_num; - nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1; - - if (udata) { - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - - if (ib_copy_from_udata(&req, udata, - sizeof(struct nes_create_cq_req))) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -EFAULT; - } - nesvnic->mcrq_ucontext = nes_ucontext; - nes_ucontext->mcrqf = req.mcrqf; - if (nes_ucontext->mcrqf) { - if (nes_ucontext->mcrqf & 0x80000000) - nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 28 + 2 * ((nes_ucontext->mcrqf & 0xf) - 1); - else if (nes_ucontext->mcrqf & 0x40000000) - nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff; - else - nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1; - nescq->mcrqf = nes_ucontext->mcrqf; - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - } - err = 1; - list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) { - if (nespbl->user_base == (unsigned long )req.user_cq_buffer) { - list_del(&nespbl->list); - err = 0; - nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n", - nespbl); - break; - } - } - if (err) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -EFAULT; - } - - pbl_entries = nespbl->pbl_size >> 3; - nescq->cq_mem_size = 0; - } else { - nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe); - nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n", - entries, nescq->cq_mem_size, nescq->hw_cq.cq_number); - - /* allocate the physical buffer space */ - mem = pci_zalloc_consistent(nesdev->pcidev, nescq->cq_mem_size, - &nescq->hw_cq.cq_pbase); - if (!mem) { - printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n"); - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -ENOMEM; - } - - nescq->hw_cq.cq_vbase = mem; - nescq->hw_cq.cq_head = 0; - } - - nescq->hw_cq.ce_handler = nes_iwarp_ce_handler; - spin_lock_init(&nescq->lock); - - /* send CreateCQ request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - NES_CQP_CQ_CHK_OVERFLOW | - NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16); - - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - - if (pbl_entries != 1) { - if (pbl_entries > 32) { - /* use 4k pbl */ - nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries); - if (nesadapter->free_4kpbl == 0) { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_free_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -ENOMEM; - } else { - opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK); - nescq->virtual_cq = 2; - nesadapter->free_4kpbl--; - } - } else { - /* use 256 byte pbl */ - nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries); - if (nesadapter->free_256pbl == 0) { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_free_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -ENOMEM; - } else { - opcode |= NES_CQP_CQ_VIRT; - nescq->virtual_cq = 1; - nesadapter->free_256pbl--; - } - } - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16))); - - if (udata) { - if (pbl_entries != 1) - u64temp = (u64)nespbl->pbl_pbase; - else - u64temp = le64_to_cpu(nespbl->pbl_vbase[0]); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX, - nes_ucontext->mmap_db_index[0]); - } else { - u64temp = (u64)nescq->hw_cq.cq_pbase; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - } - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (u64)(unsigned long)&nescq->hw_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = - cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n", - nescq->hw_cq.cq_number); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT * 2); - nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n", - nescq->hw_cq.cq_number, ret); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -EIO; - } - nes_put_cqp_request(nesdev, cqp_request); - - if (udata) { - /* free the nespbl */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - resp.cq_id = nescq->hw_cq.cq_number; - resp.cq_size = nescq->hw_cq.cq_size; - resp.mmap_db_index = 0; - if (ib_copy_to_udata(udata, &resp, - sizeof(resp) - sizeof(resp.reserved))) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return -EFAULT; - } - } - - return 0; -} - -/** - * nes_destroy_cq - */ -static void nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) -{ - struct nes_cq *nescq; - struct nes_device *nesdev; - struct nes_vnic *nesvnic; - struct nes_adapter *nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request cqp_request = {}; - unsigned long flags; - u32 opcode = 0; - - nescq = to_nescq(ib_cq); - nesvnic = to_nesvnic(ib_cq->device); - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - - nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number); - - /* Send DestroyCQ request to CQP */ - INIT_LIST_HEAD(&cqp_request.list); - init_waitqueue_head(&cqp_request.waitq); - - cqp_request.waiting = 1; - cqp_wqe = &cqp_request.cqp_wqe; - opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nescq->virtual_cq == 1) { - nesadapter->free_256pbl++; - if (nesadapter->free_256pbl > nesadapter->max_256pbl) { - printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n", - __func__, nesadapter->free_256pbl, nesadapter->max_256pbl); - } - } else if (nescq->virtual_cq == 2) { - nesadapter->free_4kpbl++; - if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) { - printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n", - __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl); - } - opcode |= NES_CQP_CQ_4KB_CHUNK; - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16))); - if (!nescq->mcrqf) - nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number); - - nes_post_cqp_request(nesdev, &cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", - nescq->hw_cq.cq_number); - wait_event_timeout(cqp_request.waitq, cqp_request.request_done, - NES_EVENT_TIMEOUT); - nes_debug( - NES_DBG_CQ, - "Destroy iWARP CQ%u completed CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nescq->hw_cq.cq_number, cqp_request.major_code, - cqp_request.minor_code); - - if (nescq->cq_mem_size) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, - nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); -} - -/** - * root_256 - */ -static u32 root_256(struct nes_device *nesdev, - struct nes_root_vpbl *root_vpbl, - struct nes_root_vpbl *new_root, - u16 pbl_count_4k) -{ - u64 leaf_pbl; - int i, j, k; - - if (pbl_count_4k == 1) { - new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, - 512, &new_root->pbl_pbase); - - if (new_root->pbl_vbase == NULL) - return 0; - - leaf_pbl = (u64)root_vpbl->pbl_pbase; - for (i = 0; i < 16; i++) { - new_root->pbl_vbase[i].pa_low = - cpu_to_le32((u32)leaf_pbl); - new_root->pbl_vbase[i].pa_high = - cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); - leaf_pbl += 256; - } - } else { - for (i = 3; i >= 0; i--) { - j = i * 16; - root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i]; - leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) + - (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high)) - << 32); - for (k = 1; k < 16; k++) { - leaf_pbl += 256; - root_vpbl->pbl_vbase[j + k].pa_low = - cpu_to_le32((u32)leaf_pbl); - root_vpbl->pbl_vbase[j + k].pa_high = - cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); - } - } - } - - return 1; -} - - -/** - * nes_reg_mr - */ -static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, - u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl, - dma_addr_t single_buffer, u16 pbl_count_4k, - u16 residual_page_count_4k, int acc, u64 *iova_start, - u16 *actual_pbl_cnt, u8 *used_4k_pbls) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - struct nes_adapter *nesadapter = nesdev->nesadapter; - uint pg_cnt = 0; - u16 pbl_count_256 = 0; - u16 pbl_count = 0; - u8 use_256_pbls = 0; - u8 use_4k_pbls = 0; - u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0; - struct nes_root_vpbl new_root = { 0, NULL, NULL }; - u32 opcode = 0; - u16 major_code; - - /* Register the region with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - if (pbl_count_4k) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - - pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k; - pbl_count_256 = (pg_cnt + 31) / 32; - if (pg_cnt <= 32) { - if (pbl_count_256 <= nesadapter->free_256pbl) - use_256_pbls = 1; - else if (pbl_count_4k <= nesadapter->free_4kpbl) - use_4k_pbls = 1; - } else if (pg_cnt <= 2048) { - if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) && - (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) { - use_4k_pbls = 1; - } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) { - use_256_pbls = 1; - use_two_level = 1; - } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { - use_4k_pbls = 1; - } - } else { - if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl) - use_4k_pbls = 1; - } - - if (use_256_pbls) { - pbl_count = pbl_count_256; - nesadapter->free_256pbl -= pbl_count + use_two_level; - } else if (use_4k_pbls) { - pbl_count = pbl_count_4k; - nesadapter->free_4kpbl -= pbl_count + use_two_level; - } else { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_debug(NES_DBG_MR, "Out of Pbls\n"); - nes_free_cqp_request(nesdev, cqp_request); - return -ENOMEM; - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - - if (use_256_pbls && use_two_level) { - if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k) == 1) { - if (new_root.pbl_pbase != 0) - root_vpbl = &new_root; - } else { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl += pbl_count_256 + use_two_level; - use_256_pbls = 0; - - if (pbl_count_4k == 1) - use_two_level = 0; - pbl_count = pbl_count_4k; - - if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { - nesadapter->free_4kpbl -= pbl_count + use_two_level; - use_4k_pbls = 1; - } - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - if (use_4k_pbls == 0) - return -ENOMEM; - } - } - - opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ | - NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; - if (acc & IB_ACCESS_LOCAL_WRITE) - opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE; - if (acc & IB_ACCESS_REMOTE_WRITE) - opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN; - if (acc & IB_ACCESS_REMOTE_READ) - opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN; - if (acc & IB_ACCESS_MW_BIND) - opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length); - - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = - cpu_to_le32((u32)(region_length >> 8) & 0xff000000); - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= - cpu_to_le32(nespd->pd_id & 0x00007fff); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - - if (pbl_count == 0) { - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer); - } else { - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8)); - - if (use_4k_pbls) - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); - } - barrier(); - - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - stag, ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - nes_put_cqp_request(nesdev, cqp_request); - - if ((!ret || major_code) && pbl_count != 0) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (use_256_pbls) - nesadapter->free_256pbl += pbl_count + use_two_level; - else if (use_4k_pbls) - nesadapter->free_4kpbl += pbl_count + use_two_level; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - if (new_root.pbl_pbase) - pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase, - new_root.pbl_pbase); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - - *actual_pbl_cnt = pbl_count + use_two_level; - *used_4k_pbls = use_4k_pbls; - return 0; -} - - -/** - * nes_reg_phys_mr - */ -struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, - int acc, u64 *iova_start) -{ - u64 region_length; - struct nes_pd *nespd = to_nespd(ib_pd); - struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_mr *nesmr; - struct ib_mr *ibmr; - struct nes_vpbl vpbl; - struct nes_root_vpbl root_vpbl; - u32 stag; - unsigned long mask; - u32 stag_index = 0; - u32 next_stag_index = 0; - u32 driver_key = 0; - int err = 0; - int ret = 0; - u16 pbl_count = 0; - u8 single_page = 1; - u8 stag_key = 0; - - region_length = 0; - vpbl.pbl_vbase = NULL; - root_vpbl.pbl_vbase = NULL; - root_vpbl.pbl_pbase = 0; - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = 0; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - if ((addr ^ *iova_start) & ~PAGE_MASK) - return ERR_PTR(-EINVAL); - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr, - &stag_index, &next_stag_index, NES_RESOURCE_PHYS_MR); - if (err) { - return ERR_PTR(err); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - /* Allocate a 4K buffer for the PBL */ - vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, - &vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n", - vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase); - if (!vpbl.pbl_vbase) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - ibmr = ERR_PTR(-ENOMEM); - kfree(nesmr); - goto reg_phys_err; - } - - - mask = !size; - - if (mask & ~PAGE_MASK) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n"); - ibmr = ERR_PTR(-EINVAL); - kfree(nesmr); - goto reg_phys_err; - } - - region_length += size; - vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)addr & PAGE_MASK); - vpbl.pbl_vbase[0].pa_high = cpu_to_le32((u32)((((u64)addr) >> 32))); - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX," - " length = 0x%016lX, index = 0x%08X\n", - stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index); - - /* Make the leaf PBL the root if only one PBL */ - root_vpbl.pbl_pbase = vpbl.pbl_pbase; - - if (single_page) { - pbl_count = 0; - } else { - pbl_count = 1; - } - ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl, - addr, pbl_count, 1, acc, iova_start, - &nesmr->pbls_used, &nesmr->pbl_4k); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MEM; - ibmr = &nesmr->ibmr; - } else { - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - } - -reg_phys_err: - /* single PBL case */ - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase); - return ibmr; -} - - -/** - * nes_get_dma_mr - */ -static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc) -{ - u64 kva = 0; - - nes_debug(NES_DBG_MR, "\n"); - - return nes_reg_phys_mr(pd, 0, 0xffffffffffULL, acc, &kva); -} - -/** - * nes_reg_user_mr - */ -static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata) -{ - u64 iova_start; - __le64 *pbl; - u64 region_length; - dma_addr_t last_dma_addr = 0; - dma_addr_t first_dma_addr = 0; - struct nes_pd *nespd = to_nespd(pd); - struct nes_vnic *nesvnic = to_nesvnic(pd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct ib_mr *ibmr = ERR_PTR(-EINVAL); - struct sg_dma_page_iter dma_iter; - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - struct nes_pbl *nespbl; - struct nes_mr *nesmr; - struct ib_umem *region; - struct nes_mem_reg_req req; - struct nes_vpbl vpbl; - struct nes_root_vpbl root_vpbl; - int page_index; - int page_count = 0; - int err, pbl_depth = 0; - int ret; - u32 stag; - u32 stag_index = 0; - u32 next_stag_index; - u32 driver_key; - u32 root_pbl_index = 0; - u32 cur_pbl_index = 0; - u32 skip_pages; - u16 pbl_count; - u8 single_page = 1; - u8 stag_key; - - region = ib_umem_get(udata, start, length, acc, 0); - if (IS_ERR(region)) { - return (struct ib_mr *)region; - } - - nes_debug( - NES_DBG_MR, - "User base = 0x%lX, Virt base = 0x%lX, length = %u, offset = %u, page size = %lu.\n", - (unsigned long)start, (unsigned long)virt, (u32)length, - ib_umem_offset(region), PAGE_SIZE); - - skip_pages = ((u32)ib_umem_offset(region)) >> 12; - - if (ib_copy_from_udata(&req, udata, sizeof(req))) { - ib_umem_release(region); - return ERR_PTR(-EFAULT); - } - nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type); - - switch (req.reg_type) { - case IWNES_MEMREG_TYPE_MEM: - pbl_depth = 0; - region_length = 0; - vpbl.pbl_vbase = NULL; - root_vpbl.pbl_vbase = NULL; - root_vpbl.pbl_pbase = 0; - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = next_stag_index & 0x70000000; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_USER_MR); - if (err) { - ib_umem_release(region); - return ERR_PTR(err); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - ib_umem_release(region); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - nesmr->region = region; - - for_each_sg_dma_page (region->sg_head.sgl, &dma_iter, region->nmap, 0) { - - region_length += PAGE_SIZE; - region_length -= skip_pages << 12; - skip_pages = 0; - if ((page_count != 0) && (page_count << 12) - (ib_umem_offset(region) & (4096 - 1)) >= region->length) - goto enough_pages; - if ((page_count & 0x01FF) == 0) { - if (page_count >= 1024 * 512) { - ib_umem_release(region); - nes_free_resource(nesadapter, - nesadapter->allocated_mrs, stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-E2BIG); - goto reg_user_mr_err; - } - if (root_pbl_index == 1) { - root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, - 8192, &root_vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", - root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); - if (!root_vpbl.pbl_vbase) { - ib_umem_release(region); - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - goto reg_user_mr_err; - } - root_vpbl.leaf_vpbl = kcalloc(1024, - sizeof(*root_vpbl.leaf_vpbl), - GFP_KERNEL); - if (!root_vpbl.leaf_vpbl) { - ib_umem_release(region); - pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, - root_vpbl.pbl_pbase); - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - goto reg_user_mr_err; - } - root_vpbl.pbl_vbase[0].pa_low = - cpu_to_le32((u32)vpbl.pbl_pbase); - root_vpbl.pbl_vbase[0].pa_high = - cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); - root_vpbl.leaf_vpbl[0] = vpbl; - } - vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, - &vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n", - vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase); - if (!vpbl.pbl_vbase) { - ib_umem_release(region); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - ibmr = ERR_PTR(-ENOMEM); - kfree(nesmr); - goto reg_user_mr_err; - } - if (1 <= root_pbl_index) { - root_vpbl.pbl_vbase[root_pbl_index].pa_low = - cpu_to_le32((u32)vpbl.pbl_pbase); - root_vpbl.pbl_vbase[root_pbl_index].pa_high = - cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); - root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; - } - root_pbl_index++; - cur_pbl_index = 0; - } - if (single_page) { - if (page_count != 0) { - if ((last_dma_addr + 4096) != sg_page_iter_dma_address(&dma_iter)) - single_page = 0; - last_dma_addr = sg_page_iter_dma_address(&dma_iter); - } else { - first_dma_addr = sg_page_iter_dma_address(&dma_iter); - last_dma_addr = first_dma_addr; - } - } - - vpbl.pbl_vbase[cur_pbl_index].pa_low = - cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter))); - vpbl.pbl_vbase[cur_pbl_index].pa_high = - cpu_to_le32((u32)((u64)(sg_page_iter_dma_address(&dma_iter)))); - cur_pbl_index++; - page_count++; - } - -enough_pages: - nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," - " stag_key=0x%08x\n", - stag_index, driver_key, stag_key); - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - iova_start = virt; - /* Make the leaf PBL the root if only one PBL */ - if (root_pbl_index == 1) { - root_vpbl.pbl_pbase = vpbl.pbl_pbase; - } - - if (single_page) { - pbl_count = 0; - } else { - pbl_count = root_pbl_index; - first_dma_addr = 0; - } - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X," - " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n", - stag, (unsigned int)iova_start, - (unsigned int)region_length, stag_index, - (unsigned long long)region->length, pbl_count); - ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl, - first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, - &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k); - - nes_debug(NES_DBG_MR, "ret=%d\n", ret); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MEM; - ibmr = &nesmr->ibmr; - } else { - ib_umem_release(region); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - } - -reg_user_mr_err: - /* free the resources */ - if (root_pbl_index == 1) { - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - } else { - for (page_index=0; page_indexpcidev, 4096, - root_vpbl.leaf_vpbl[page_index].pbl_vbase, - root_vpbl.leaf_vpbl[page_index].pbl_pbase); - } - kfree(root_vpbl.leaf_vpbl); - pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, - root_vpbl.pbl_pbase); - } - - nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr); - - return ibmr; - case IWNES_MEMREG_TYPE_QP: - case IWNES_MEMREG_TYPE_CQ: - if (!region->length) { - nes_debug(NES_DBG_MR, "Unable to register zero length region for CQ\n"); - ib_umem_release(region); - return ERR_PTR(-EINVAL); - } - nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL); - if (!nespbl) { - ib_umem_release(region); - return ERR_PTR(-ENOMEM); - } - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - ib_umem_release(region); - kfree(nespbl); - return ERR_PTR(-ENOMEM); - } - nesmr->region = region; - pbl_depth = region->length >> 12; - pbl_depth += (region->length & (4096-1)) ? 1 : 0; - nespbl->pbl_size = pbl_depth*sizeof(u64); - if (req.reg_type == IWNES_MEMREG_TYPE_QP) { - nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory"); - } else { - nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory"); - } - - nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n", - nespbl->pbl_size, pbl_depth); - pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size, - &nespbl->pbl_pbase); - if (!pbl) { - ib_umem_release(region); - kfree(nesmr); - kfree(nespbl); - nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n"); - return ERR_PTR(-ENOMEM); - } - - nespbl->pbl_vbase = (u64 *)pbl; - nespbl->user_base = start; - nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx," - " pbl_vbase=%p user_base=0x%lx\n", - nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, - (void *) nespbl->pbl_vbase, nespbl->user_base); - - nespbl->page = sg_page(region->sg_head.sgl); - for_each_sg_dma_page(region->sg_head.sgl, &dma_iter, region->nmap, 0) { - ((__le32 *)pbl)[0] = cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter))); - ((__le32 *)pbl)[1] = cpu_to_le32(((u64)(sg_page_iter_dma_address(&dma_iter)))>>32); - nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, - (unsigned long long)*pbl, - le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); - pbl++; - } - - if (req.reg_type == IWNES_MEMREG_TYPE_QP) { - list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); - } else { - list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list); - } - nesmr->ibmr.rkey = -1; - nesmr->ibmr.lkey = -1; - nesmr->mode = req.reg_type; - return &nesmr->ibmr; - } - - ib_umem_release(region); - return ERR_PTR(-ENOSYS); -} - - -/** - * nes_dereg_mr - */ -static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) -{ - struct nes_mr *nesmr = to_nesmr(ib_mr); - struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - u16 major_code; - u16 minor_code; - - - if (nesmr->pages) - pci_free_consistent(nesdev->pcidev, - nesmr->max_pages * sizeof(u64), - nesmr->pages, - nesmr->paddr); - - if (nesmr->region) { - ib_umem_release(nesmr->region); - } - if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) { - kfree(nesmr); - return 0; - } - - /* Deallocate the region with the adapter */ - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey); - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X\n", - ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code); - - major_code = cqp_request->major_code; - minor_code = cqp_request->minor_code; - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) { - nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag," - " ib_mr=%p, rkey = 0x%08X\n", - ib_mr, ib_mr->rkey); - return -ETIME; - } else if (major_code) { - nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting" - " to destroy STag, ib_mr=%p, rkey = 0x%08X\n", - major_code, minor_code, ib_mr, ib_mr->rkey); - return -EIO; - } - - if (nesmr->pbls_used != 0) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nesmr->pbl_4k) { - nesadapter->free_4kpbl += nesmr->pbls_used; - if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) - printk(KERN_ERR PFX "free 4KB PBLs(%u) has " - "exceeded the max(%u)\n", - nesadapter->free_4kpbl, - nesadapter->max_4kpbl); - } else { - nesadapter->free_256pbl += nesmr->pbls_used; - if (nesadapter->free_256pbl > nesadapter->max_256pbl) - printk(KERN_ERR PFX "free 256B PBLs(%u) has " - "exceeded the max(%u)\n", - nesadapter->free_256pbl, - nesadapter->max_256pbl); - } - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - (ib_mr->rkey & 0x0fffff00) >> 8); - - kfree(nesmr); - - return 0; -} - - -/** - * show_rev - */ -static ssize_t hw_rev_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct nes_ib_device *nesibdev = - rdma_device_to_drv_device(dev, struct nes_ib_device, ibdev); - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev); -} -static DEVICE_ATTR_RO(hw_rev); - -/** - * show_hca - */ -static ssize_t hca_type_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "NES020\n"); -} -static DEVICE_ATTR_RO(hca_type); - -/** - * show_board - */ -static ssize_t board_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "%.*s\n", 32, "NES020 Board ID"); -} -static DEVICE_ATTR_RO(board_id); - -static struct attribute *nes_dev_attributes[] = { - &dev_attr_hw_rev.attr, - &dev_attr_hca_type.attr, - &dev_attr_board_id.attr, - NULL -}; - -static const struct attribute_group nes_attr_group = { - .attrs = nes_dev_attributes, -}; - -/** - * nes_query_qp - */ -static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_qp_init_attr *init_attr) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - nes_debug(NES_DBG_QP, "\n"); - - attr->qp_access_flags = 0; - attr->cap.max_send_wr = nesqp->hwqp.sq_size; - attr->cap.max_recv_wr = nesqp->hwqp.rq_size; - attr->cap.max_recv_sge = 1; - if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) - attr->cap.max_inline_data = 0; - else - attr->cap.max_inline_data = 64; - - init_attr->event_handler = nesqp->ibqp.event_handler; - init_attr->qp_context = nesqp->ibqp.qp_context; - init_attr->send_cq = nesqp->ibqp.send_cq; - init_attr->recv_cq = nesqp->ibqp.recv_cq; - init_attr->srq = nesqp->ibqp.srq; - init_attr->cap = attr->cap; - - return 0; -} - - -/** - * nes_hw_modify_qp - */ -int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, - u32 next_iwarp_state, u32 termlen, u32 wait_completion) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - /* struct iw_cm_id *cm_id = nesqp->cm_id; */ - /* struct iw_cm_event cm_event; */ - struct nes_cqp_request *cqp_request; - int ret; - u16 major_code; - - nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - if (wait_completion) { - cqp_request->waiting = 1; - } else { - cqp_request->waiting = 0; - } - cqp_wqe = &cqp_request->cqp_wqe; - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state); - nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n", - next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])); - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase); - - /* If sending a terminate message, fill in the length (in words) */ - if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) && - !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) { - termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen); - } - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - if (wait_completion) { - /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n", - nesqp->hwqp.qp_id); */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, " - "CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - if (major_code) { - nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed" - "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n", - nesqp->hwqp.qp_id, cqp_request->major_code, - cqp_request->minor_code, next_iwarp_state); - } - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - else - return 0; - } else { - return 0; - } -} - - -/** - * nes_modify_qp - */ -int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - /* u32 cqp_head; */ - /* u32 counter; */ - u32 next_iwarp_state = 0; - int err; - unsigned long qplockflags; - int ret; - u16 original_last_aeq; - u8 issue_modify_qp = 0; - u8 dont_wait = 0; - - nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u," - " iwarp_state=0x%X, refcount=%d\n", - nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state, - nesqp->iwarp_state, atomic_read(&nesqp->refcount)); - - spin_lock_irqsave(&nesqp->lock, qplockflags); - - nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X," - " QP Access Flags=0x%X, attr_mask = 0x%0x\n", - nesqp->hwqp.qp_id, nesqp->hw_iwarp_state, - nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask); - - if (attr_mask & IB_QP_STATE) { - switch (attr->qp_state) { - case IB_QPS_INIT: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; - issue_modify_qp = 1; - break; - case IB_QPS_RTR: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; - issue_modify_qp = 1; - break; - case IB_QPS_RTS: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - if (nesqp->cm_id == NULL) { - nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n", - nesqp->hwqp.qp_id ); - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS; - if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS) - next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID | - NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID; - issue_modify_qp = 1; - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS; - nesqp->hte_added = 1; - break; - case IB_QPS_SQD: - issue_modify_qp = 1; - nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n", - nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail); - if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return 0; - } else { - if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { - nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" - " ignored due to current iWARP state\n", - nesqp->hwqp.qp_id); - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) { - nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" - " already done based on hw state.\n", - nesqp->hwqp.qp_id); - issue_modify_qp = 0; - } - switch (nesqp->hw_iwarp_state) { - case NES_AEQE_IWARP_STATE_CLOSING: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - break; - case NES_AEQE_IWARP_STATE_TERMINATE: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; - break; - case NES_AEQE_IWARP_STATE_ERROR: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - break; - default: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; - break; - } - } - break; - case IB_QPS_SQE: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; - issue_modify_qp = 1; - break; - case IB_QPS_ERR: - case IB_QPS_RESET: - if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n", - nesqp->hwqp.qp_id); - if (nesqp->term_flags) - del_timer(&nesqp->terminate_timer); - - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ - if (nesqp->hte_added) { - nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n"); - next_iwarp_state |= NES_CQP_QP_DEL_HTE; - nesqp->hte_added = 0; - } - if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && - (nesdev->iw_status) && - (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { - next_iwarp_state |= NES_CQP_QP_RESET; - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n", - nesqp->hwqp.qp_id, nesqp->hw_tcp_state); - dont_wait = 1; - } - issue_modify_qp = 1; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR; - break; - default: - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - break; - } - - nesqp->ibqp_state = attr->qp_state; - nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; - nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", - nesqp->iwarp_state); - } - - if (attr_mask & IB_QP_ACCESS_FLAGS) { - if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | - NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_MW_BIND) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN); - issue_modify_qp = 1; - } - - if (nesqp->user_mode) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | - NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - } - - original_last_aeq = nesqp->last_aeq; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - - nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp); - - ret = 0; - - - if (issue_modify_qp) { - nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n"); - ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1); - if (ret) - nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)" - " failed for QP%u.\n", - next_iwarp_state, nesqp->hwqp.qp_id); - - } - - if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) { - nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d)," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - if (!ret || original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) { - if (dont_wait) { - if (nesqp->cm_id && nesqp->hw_tcp_state != 0) { - nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d)," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - /* this one is for the cm_disconnect thread */ - spin_lock_irqsave(&nesqp->lock, qplockflags); - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_cm_disconn(nesqp); - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - } - } else { - spin_lock_irqsave(&nesqp->lock, qplockflags); - if (nesqp->cm_id) { - /* These two are for the timer thread */ - if (atomic_inc_return(&nesqp->close_timer_started) == 1) { - nesqp->cm_id->add_ref(nesqp->cm_id); - nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X, scheduling timer.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0); - } - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - } else { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - } - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - - err = 0; - - nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - - return err; -} - -static inline void -fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, const struct ib_send_wr *ib_wr, - u32 uselkey) -{ - int sge_index; - int total_payload_length = 0; - for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) { - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4), - ib_wr->sg_list[sge_index].length); - if (uselkey) - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), - (ib_wr->sg_list[sge_index].lkey)); - else - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0); - - total_payload_length += ib_wr->sg_list[sge_index].length; - } - nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n", - total_payload_length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - total_payload_length); -} - -/** - * nes_post_send - */ -static int nes_post_send(struct ib_qp *ibqp, const struct ib_send_wr *ib_wr, - const struct ib_send_wr **bad_wr) -{ - u64 u64temp; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_hw_qp_wqe *wqe; - int err = 0; - u32 qsize = nesqp->hwqp.sq_size; - u32 head; - u32 wqe_misc = 0; - u32 wqe_count = 0; - u32 counter; - - if (nesqp->ibqp_state > IB_QPS_RTS) { - err = -EINVAL; - goto out; - } - - spin_lock_irqsave(&nesqp->lock, flags); - - head = nesqp->hwqp.sq_head; - - while (ib_wr) { - /* Check for QP error */ - if (nesqp->term_flags) { - err = -EINVAL; - break; - } - - /* Check for SQ overflow */ - if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { - err = -ENOMEM; - break; - } - - wqe = &nesqp->hwqp.sq_vbase[head]; - /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n", - nesqp->hwqp.qp_id, wqe, head); */ - nes_fill_init_qp_wqe(wqe, nesqp, head); - u64temp = (u64)(ib_wr->wr_id); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, - u64temp); - switch (ib_wr->opcode) { - case IB_WR_SEND: - case IB_WR_SEND_WITH_INV: - if (IB_WR_SEND == ib_wr->opcode) { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - wqe_misc = NES_IWARP_SQ_OP_SENDSE; - else - wqe_misc = NES_IWARP_SQ_OP_SEND; - } else { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - wqe_misc = NES_IWARP_SQ_OP_SENDSEINV; - else - wqe_misc = NES_IWARP_SQ_OP_SENDINV; - - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, - ib_wr->ex.invalidate_rkey); - } - - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - err = -EINVAL; - break; - } - - if (ib_wr->send_flags & IB_SEND_FENCE) - wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; - - if ((ib_wr->send_flags & IB_SEND_INLINE) && - ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && - (ib_wr->sg_list[0].length <= 64)) { - memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], - (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - ib_wr->sg_list[0].length); - wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; - } else { - fill_wqe_sg_send(wqe, ib_wr, 1); - } - - break; - case IB_WR_RDMA_WRITE: - wqe_misc = NES_IWARP_SQ_OP_RDMAW; - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n", - ib_wr->num_sge, nesdev->nesadapter->max_sge); - err = -EINVAL; - break; - } - - if (ib_wr->send_flags & IB_SEND_FENCE) - wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; - - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, - rdma_wr(ib_wr)->rkey); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, - rdma_wr(ib_wr)->remote_addr); - - if ((ib_wr->send_flags & IB_SEND_INLINE) && - ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && - (ib_wr->sg_list[0].length <= 64)) { - memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], - (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - ib_wr->sg_list[0].length); - wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; - } else { - fill_wqe_sg_send(wqe, ib_wr, 1); - } - - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]; - break; - case IB_WR_RDMA_READ: - case IB_WR_RDMA_READ_WITH_INV: - /* iWARP only supports 1 sge for RDMA reads */ - if (ib_wr->num_sge > 1) { - nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n", - ib_wr->num_sge); - err = -EINVAL; - break; - } - if (ib_wr->opcode == IB_WR_RDMA_READ) { - wqe_misc = NES_IWARP_SQ_OP_RDMAR; - } else { - wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV; - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, - ib_wr->ex.invalidate_rkey); - } - - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, - rdma_wr(ib_wr)->remote_addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, - rdma_wr(ib_wr)->rkey); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX, - ib_wr->sg_list->length); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, - ib_wr->sg_list->addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX, - ib_wr->sg_list->lkey); - break; - case IB_WR_LOCAL_INV: - wqe_misc = NES_IWARP_SQ_OP_LOCINV; - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX, - ib_wr->ex.invalidate_rkey); - break; - case IB_WR_REG_MR: - { - struct nes_mr *mr = to_nesmr(reg_wr(ib_wr)->mr); - int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size); - int flags = reg_wr(ib_wr)->access; - - if (mr->npages > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) { - nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n"); - err = -EINVAL; - break; - } - wqe_misc = NES_IWARP_SQ_OP_FAST_REG; - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX, - mr->ibmr.iova); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX, - lower_32_bits(mr->ibmr.length)); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX, 0); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX, - reg_wr(ib_wr)->key); - - if (page_shift == 12) { - wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K; - } else if (page_shift == 21) { - wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M; - } else { - nes_debug(NES_DBG_IW_TX, "Invalid page shift," - " ib_wr=%u, max=1\n", ib_wr->num_sge); - err = -EINVAL; - break; - } - - /* Set access_flags */ - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ; - if (flags & IB_ACCESS_LOCAL_WRITE) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE; - - if (flags & IB_ACCESS_REMOTE_WRITE) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE; - - if (flags & IB_ACCESS_REMOTE_READ) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ; - - if (flags & IB_ACCESS_MW_BIND) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND; - - /* Fill in PBL info: */ - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX, - mr->paddr); - - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX, - mr->npages * 8); - - nes_debug(NES_DBG_IW_TX, "SQ_REG_MR: iova_start: %llx, " - "length: %lld, rkey: %0x, pgl_paddr: %llx, " - "page_list_len: %u, wqe_misc: %x\n", - (unsigned long long) mr->ibmr.iova, - mr->ibmr.length, - reg_wr(ib_wr)->key, - (unsigned long long) mr->paddr, - mr->npages, - wqe_misc); - break; - } - default: - /* error */ - err = -EINVAL; - break; - } - - if (err) - break; - - if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all) - wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; - - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc); - - ib_wr = ib_wr->next; - head++; - wqe_count++; - if (head >= qsize) - head = 0; - - } - - nesqp->hwqp.sq_head = head; - barrier(); - while (wqe_count) { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs + NES_WQE_ALLOC, - (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id); - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - -out: - if (err) - *bad_wr = ib_wr; - return err; -} - - -/** - * nes_post_recv - */ -static int nes_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr, - const struct ib_recv_wr **bad_wr) -{ - u64 u64temp; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_hw_qp_wqe *wqe; - int err = 0; - int sge_index; - u32 qsize = nesqp->hwqp.rq_size; - u32 head; - u32 wqe_count = 0; - u32 counter; - u32 total_payload_length; - - if (nesqp->ibqp_state > IB_QPS_RTS) { - err = -EINVAL; - goto out; - } - - spin_lock_irqsave(&nesqp->lock, flags); - - head = nesqp->hwqp.rq_head; - - while (ib_wr) { - /* Check for QP error */ - if (nesqp->term_flags) { - err = -EINVAL; - break; - } - - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - err = -EINVAL; - break; - } - /* Check for RQ overflow */ - if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) { - err = -ENOMEM; - break; - } - - nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge); - wqe = &nesqp->hwqp.rq_vbase[head]; - - /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n", - nesqp->hwqp.qp_id, wqe, head); */ - nes_fill_init_qp_wqe(wqe, nesqp, head); - u64temp = (u64)(ib_wr->wr_id); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, - u64temp); - total_payload_length = 0; - for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].length); - set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].lkey); - - total_payload_length += ib_wr->sg_list[sge_index].length; - } - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX, - total_payload_length); - - ib_wr = ib_wr->next; - head++; - wqe_count++; - if (head >= qsize) - head = 0; - } - - nesqp->hwqp.rq_head = head; - barrier(); - while (wqe_count) { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id); - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - -out: - if (err) - *bad_wr = ib_wr; - return err; -} - -/** - * nes_drain_sq - drain sq - * @ibqp: pointer to ibqp - */ -static void nes_drain_sq(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - if (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head) - wait_for_completion(&nesqp->sq_drained); -} - -/** - * nes_drain_rq - drain rq - * @ibqp: pointer to ibqp - */ -static void nes_drain_rq(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - if (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head) - wait_for_completion(&nesqp->rq_drained); -} - -/** - * nes_poll_cq - */ -static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) -{ - u64 u64temp; - u64 wrid; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_cq *nescq = to_nescq(ibcq); - struct nes_qp *nesqp; - struct nes_hw_cqe cqe; - u32 head; - u32 wq_tail = 0; - u32 cq_size; - u32 cqe_count = 0; - u32 wqe_index; - u32 u32temp; - u32 move_cq_head = 1; - u32 err_code; - - nes_debug(NES_DBG_CQ, "\n"); - - spin_lock_irqsave(&nescq->lock, flags); - - head = nescq->hw_cq.cq_head; - cq_size = nescq->hw_cq.cq_size; - - while (cqe_count < num_entries) { - if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & - NES_CQE_VALID) == 0) - break; - - /* - * Make sure we read CQ entry contents *after* - * we've checked the valid bit. - */ - rmb(); - - cqe = nescq->hw_cq.cq_vbase[head]; - u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1); - u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); - /* parse CQE, get completion context from WQE (either rq or sq) */ - u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | - ((u64)u32temp); - - if (u64temp) { - nesqp = (struct nes_qp *)(unsigned long)u64temp; - memset(entry, 0, sizeof *entry); - if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { - entry->status = IB_WC_SUCCESS; - } else { - err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); - if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) { - entry->status = err_code & 0x0000ffff; - - /* The rest of the cqe's will be marked as flushed */ - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = - cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | - NES_IWARP_CQE_MINOR_FLUSH); - } else - entry->status = IB_WC_WR_FLUSH_ERR; - } - - entry->qp = &nesqp->ibqp; - entry->src_qp = nesqp->hwqp.qp_id; - - if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { - if (nesqp->skip_lsmm) { - nesqp->skip_lsmm = 0; - nesqp->hwqp.sq_tail++; - } - - /* Working on a SQ Completion*/ - wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) | - ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX]))); - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); - - switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { - case NES_IWARP_SQ_OP_RDMAW: - nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n"); - entry->opcode = IB_WC_RDMA_WRITE; - break; - case NES_IWARP_SQ_OP_RDMAR: - nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n"); - entry->opcode = IB_WC_RDMA_READ; - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); - break; - case NES_IWARP_SQ_OP_SENDINV: - case NES_IWARP_SQ_OP_SENDSEINV: - case NES_IWARP_SQ_OP_SEND: - case NES_IWARP_SQ_OP_SENDSE: - nes_debug(NES_DBG_CQ, "Operation = Send.\n"); - entry->opcode = IB_WC_SEND; - break; - case NES_IWARP_SQ_OP_LOCINV: - entry->opcode = IB_WC_LOCAL_INV; - break; - case NES_IWARP_SQ_OP_FAST_REG: - entry->opcode = IB_WC_REG_MR; - break; - } - - nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); - if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) { - move_cq_head = 0; - wq_tail = nesqp->hwqp.sq_tail; - } - } else { - /* Working on a RQ Completion*/ - entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); - wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | - ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); - entry->opcode = IB_WC_RECV; - - nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); - if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) { - move_cq_head = 0; - wq_tail = nesqp->hwqp.rq_tail; - } - } - - if (nesqp->iwarp_state > NES_CQP_QP_IWARP_STATE_RTS) { - if (nesqp->hwqp.sq_tail == nesqp->hwqp.sq_head) - complete(&nesqp->sq_drained); - if (nesqp->hwqp.rq_tail == nesqp->hwqp.rq_head) - complete(&nesqp->rq_drained); - } - - entry->wr_id = wrid; - entry++; - cqe_count++; - } - - if (move_cq_head) { - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; - if (++head >= cq_size) - head = 0; - nescq->polled_completions++; - - if ((nescq->polled_completions > (cq_size / 2)) || - (nescq->polled_completions == 255)) { - nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" - " are pending %u of %u.\n", - nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; - } - } else { - /* Update the wqe index and set status to flush */ - wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail; - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = - cpu_to_le32(wqe_index); - move_cq_head = 1; /* ready for next pass */ - } - } - - if (nescq->polled_completions) { - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; - } - - nescq->hw_cq.cq_head = head; - nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n", - cqe_count, nescq->hw_cq.cq_number); - - spin_unlock_irqrestore(&nescq->lock, flags); - - return cqe_count; -} - - -/** - * nes_req_notify_cq - */ -static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags) - { - struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_cq *nescq = to_nescq(ibcq); - u32 cq_arm; - - nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n", - nescq->hw_cq.cq_number); - - cq_arm = nescq->hw_cq.cq_number; - if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP) - cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT; - else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) - cq_arm |= NES_CQE_ALLOC_NOTIFY_SE; - else - return -EINVAL; - - nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - - return 0; -} - -static int nes_port_immutable(struct ib_device *ibdev, u8 port_num, - struct ib_port_immutable *immutable) -{ - struct ib_port_attr attr; - int err; - - immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; - - err = nes_query_port(ibdev, port_num, &attr); - if (err) - return err; - - immutable->pkey_tbl_len = attr.pkey_tbl_len; - immutable->gid_tbl_len = attr.gid_tbl_len; - - return 0; -} - -static void get_dev_fw_str(struct ib_device *dev, char *str) -{ - struct nes_ib_device *nesibdev = - container_of(dev, struct nes_ib_device, ibdev); - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - nes_debug(NES_DBG_INIT, "\n"); - snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", - (nesvnic->nesdev->nesadapter->firmware_version >> 16), - (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff)); -} - -static const struct ib_device_ops nes_dev_ops = { - .owner = THIS_MODULE, - .driver_id = RDMA_DRIVER_NES, - /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ - .uverbs_abi_ver = NES_ABI_USERSPACE_VER, - - .alloc_mr = nes_alloc_mr, - .alloc_mw = nes_alloc_mw, - .alloc_pd = nes_alloc_pd, - .alloc_ucontext = nes_alloc_ucontext, - .create_cq = nes_create_cq, - .create_qp = nes_create_qp, - .dealloc_mw = nes_dealloc_mw, - .dealloc_pd = nes_dealloc_pd, - .dealloc_ucontext = nes_dealloc_ucontext, - .dereg_mr = nes_dereg_mr, - .destroy_cq = nes_destroy_cq, - .destroy_qp = nes_destroy_qp, - .drain_rq = nes_drain_rq, - .drain_sq = nes_drain_sq, - .get_dev_fw_str = get_dev_fw_str, - .get_dma_mr = nes_get_dma_mr, - .get_port_immutable = nes_port_immutable, - .iw_accept = nes_accept, - .iw_add_ref = nes_add_ref, - .iw_connect = nes_connect, - .iw_create_listen = nes_create_listen, - .iw_destroy_listen = nes_destroy_listen, - .iw_get_qp = nes_get_qp, - .iw_reject = nes_reject, - .iw_rem_ref = nes_rem_ref, - .map_mr_sg = nes_map_mr_sg, - .mmap = nes_mmap, - .modify_qp = nes_modify_qp, - .poll_cq = nes_poll_cq, - .post_recv = nes_post_recv, - .post_send = nes_post_send, - .query_device = nes_query_device, - .query_gid = nes_query_gid, - .query_pkey = nes_query_pkey, - .query_port = nes_query_port, - .query_qp = nes_query_qp, - .reg_user_mr = nes_reg_user_mr, - .req_notify_cq = nes_req_notify_cq, - INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd), - INIT_RDMA_OBJ_SIZE(ib_cq, nes_cq, ibcq), - INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext), -}; - -/** - * nes_init_ofa_device - */ -struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) -{ - struct nes_ib_device *nesibdev; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - - nesibdev = ib_alloc_device(nes_ib_device, ibdev); - if (nesibdev == NULL) { - return NULL; - } - - nesibdev->ibdev.node_type = RDMA_NODE_RNIC; - memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid)); - memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6); - - nesibdev->ibdev.uverbs_cmd_mask = - (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | - (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | - (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | - (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_REG_MR) | - (1ull << IB_USER_VERBS_CMD_DEREG_MR) | - (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | - (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_AH) | - (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | - (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_QP) | - (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | - (1ull << IB_USER_VERBS_CMD_POLL_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | - (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | - (1ull << IB_USER_VERBS_CMD_BIND_MW) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | - (1ull << IB_USER_VERBS_CMD_POST_RECV) | - (1ull << IB_USER_VERBS_CMD_POST_SEND); - - nesibdev->ibdev.phys_port_cnt = 1; - nesibdev->ibdev.num_comp_vectors = 1; - nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev; - - ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops); - memcpy(nesibdev->ibdev.iw_ifname, netdev->name, - sizeof(nesibdev->ibdev.iw_ifname)); - - return nesibdev; -} - - -/** - * nes_handle_delayed_event - */ -static void nes_handle_delayed_event(struct timer_list *t) -{ - struct nes_vnic *nesvnic = from_timer(nesvnic, t, event_timer); - - if (nesvnic->delayed_event != nesvnic->last_dispatched_event) { - struct ib_event event; - - event.device = &nesvnic->nesibdev->ibdev; - if (!event.device) - goto stop_timer; - event.event = nesvnic->delayed_event; - event.element.port_num = nesvnic->logical_port + 1; - ib_dispatch_event(&event); - } - -stop_timer: - nesvnic->event_timer.function = NULL; -} - - -void nes_port_ibevent(struct nes_vnic *nesvnic) -{ - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - struct nes_device *nesdev = nesvnic->nesdev; - struct ib_event event; - event.device = &nesibdev->ibdev; - event.element.port_num = nesvnic->logical_port + 1; - event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; - - if (!nesvnic->event_timer.function) { - ib_dispatch_event(&event); - nesvnic->last_dispatched_event = event.event; - nesvnic->event_timer.function = nes_handle_delayed_event; - nesvnic->event_timer.expires = jiffies + NES_EVENT_DELAY; - add_timer(&nesvnic->event_timer); - } else { - mod_timer(&nesvnic->event_timer, jiffies + NES_EVENT_DELAY); - } - nesvnic->delayed_event = event.event; -} - - -/** - * nes_destroy_ofa_device - */ -void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) -{ - nes_unregister_ofa_device(nesibdev); - - ib_dealloc_device(&nesibdev->ibdev); -} - - -/** - * nes_register_ofa_device - */ -int nes_register_ofa_device(struct nes_ib_device *nesibdev) -{ - struct nes_vnic *nesvnic = nesibdev->nesvnic; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - int ret; - - rdma_set_device_sysfs_group(&nesvnic->nesibdev->ibdev, &nes_attr_group); - ret = ib_register_device(&nesvnic->nesibdev->ibdev, "nes%d"); - if (ret) { - return ret; - } - - /* Get the resources allocated to this device */ - nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count; - nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count; - nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count; - nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count; - - nesvnic->of_device_registered = 1; - - return 0; -} - - -/** - * nes_unregister_ofa_device - */ -static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev) -{ - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - if (nesvnic->of_device_registered) - ib_unregister_device(&nesibdev->ibdev); - - nesvnic->of_device_registered = 0; -} diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h deleted file mode 100644 index 114a9b59fefd..000000000000 --- a/drivers/infiniband/hw/nes/nes_verbs.h +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef NES_VERBS_H -#define NES_VERBS_H - -struct nes_device; - -#define NES_MAX_USER_DB_REGIONS 4096 -#define NES_MAX_USER_WQ_REGIONS 4096 - -#define NES_TERM_SENT 0x01 -#define NES_TERM_RCVD 0x02 -#define NES_TERM_DONE 0x04 - -struct nes_ucontext { - struct ib_ucontext ibucontext; - struct nes_device *nesdev; - unsigned long mmap_wq_offset; - unsigned long mmap_cq_offset; /* to be removed */ - int index; /* rnic index (minor) */ - unsigned long allocated_doorbells[BITS_TO_LONGS(NES_MAX_USER_DB_REGIONS)]; - u16 mmap_db_index[NES_MAX_USER_DB_REGIONS]; - u16 first_free_db; - unsigned long allocated_wqs[BITS_TO_LONGS(NES_MAX_USER_WQ_REGIONS)]; - struct nes_qp *mmap_nesqp[NES_MAX_USER_WQ_REGIONS]; - u16 first_free_wq; - struct list_head cq_reg_mem_list; - struct list_head qp_reg_mem_list; - u32 mcrqf; -}; - -struct nes_pd { - struct ib_pd ibpd; - u16 pd_id; - atomic_t sqp_count; - u16 mmap_db_index; -}; - -struct nes_mr { - union { - struct ib_mr ibmr; - struct ib_mw ibmw; - struct ib_fmr ibfmr; - }; - struct ib_umem *region; - u16 pbls_used; - u8 mode; - u8 pbl_4k; - __le64 *pages; - dma_addr_t paddr; - u32 max_pages; - u32 npages; -}; - -struct nes_hw_pb { - __le32 pa_low; - __le32 pa_high; -}; - -struct nes_vpbl { - dma_addr_t pbl_pbase; - struct nes_hw_pb *pbl_vbase; -}; - -struct nes_root_vpbl { - dma_addr_t pbl_pbase; - struct nes_hw_pb *pbl_vbase; - struct nes_vpbl *leaf_vpbl; -}; - -struct nes_fmr { - struct nes_mr nesmr; - u32 leaf_pbl_cnt; - struct nes_root_vpbl root_vpbl; - struct ib_qp *ib_qp; - int access_rights; - struct ib_fmr_attr attr; -}; - -struct nes_av; - -struct nes_cq { - struct ib_cq ibcq; - struct nes_hw_cq hw_cq; - u32 polled_completions; - u32 cq_mem_size; - spinlock_t lock; - u8 virtual_cq; - u8 pad[3]; - u32 mcrqf; -}; - -struct nes_wq { - spinlock_t lock; -}; - -struct disconn_work { - struct work_struct work; - struct nes_qp *nesqp; -}; - -struct iw_cm_id; -struct ietf_mpa_frame; - -struct nes_qp { - struct ib_qp ibqp; - void *allocated_buffer; - struct iw_cm_id *cm_id; - struct nes_cq *nesscq; - struct nes_cq *nesrcq; - struct nes_pd *nespd; - void *cm_node; /* handle of the node this QP is associated with */ - void *ietf_frame; - u8 ietf_frame_size; - dma_addr_t ietf_frame_pbase; - struct ib_mr *lsmm_mr; - struct nes_hw_qp hwqp; - struct work_struct work; - enum ib_qp_state ibqp_state; - u32 iwarp_state; - u32 hte_index; - u32 last_aeq; - u32 qp_mem_size; - atomic_t refcount; - atomic_t close_timer_started; - u32 mmap_sq_db_index; - u32 mmap_rq_db_index; - spinlock_t lock; - spinlock_t pau_lock; - struct nes_qp_context *nesqp_context; - dma_addr_t nesqp_context_pbase; - void *pbl_vbase; - dma_addr_t pbl_pbase; - struct page *page; - struct timer_list terminate_timer; - enum ib_event_type terminate_eventtype; - struct sk_buff_head pau_list; - u32 pau_rcv_nxt; - u16 active_conn:1; - u16 skip_lsmm:1; - u16 user_mode:1; - u16 hte_added:1; - u16 flush_issued:1; - u16 destroyed:1; - u16 sig_all:1; - u16 pau_mode:1; - u16 rsvd:8; - u16 private_data_len; - u16 term_sq_flush_code; - u16 term_rq_flush_code; - u8 hw_iwarp_state; - u8 hw_tcp_state; - u8 term_flags; - u8 sq_kmapped; - u8 pau_busy; - u8 pau_pending; - u8 pau_state; - __u64 nesuqp_addr; - struct completion sq_drained; - struct completion rq_drained; -}; - -struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, - u64 addr, u64 size, int acc, u64 *iova_start); - -#endif /* NES_VERBS_H */ -- cgit v1.2.3 From 8f71bb0030b8816f57be142f95b3c7189c6eaf4c Mon Sep 17 00:00:00 2001 From: Jason Gunthorpe Date: Thu, 13 Jun 2019 21:38:19 -0300 Subject: RDMA: Report available cdevs through RDMA_NLDEV_CMD_GET_CHARDEV Update the struct ib_client for all modules exporting cdevs related to the ibdevice to also implement RDMA_NLDEV_CMD_GET_CHARDEV. All cdevs are now autoloadable and discoverable by userspace over netlink instead of relying on sysfs. uverbs also exposes the DRIVER_ID for drivers that are able to support driver id binding in rdma-core. Signed-off-by: Jason Gunthorpe Reviewed-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/cxgb3/iwch_provider.c | 1 + drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/mthca/mthca_provider.c | 1 + 3 files changed, 3 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index acba96f289cc..810fa96af2e9 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -1230,6 +1230,7 @@ static const struct ib_device_ops iwch_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_CXGB3, .uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION, + .uverbs_no_driver_id_binding = 1, .alloc_hw_stats = iwch_alloc_stats, .alloc_mr = iwch_alloc_mr, diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 3e45b119b0eb..c0e819ed8c9b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -417,6 +417,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_HNS, .uverbs_abi_ver = 1, + .uverbs_no_driver_id_binding = 1, .add_gid = hns_roce_add_gid, .alloc_pd = hns_roce_alloc_pd, diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index efd4e3d13ae2..d97124bee703 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -1147,6 +1147,7 @@ static const struct ib_device_ops mthca_dev_ops = { .owner = THIS_MODULE, .driver_id = RDMA_DRIVER_MTHCA, .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, + .uverbs_no_driver_id_binding = 1, .alloc_pd = mthca_alloc_pd, .alloc_ucontext = mthca_alloc_ucontext, -- cgit v1.2.3 From 5a3113d19cb0ec06fd0d068a2f2860786a770e4f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 17 Jun 2019 16:01:38 +0200 Subject: IB/hfi1: Spelling s/statisfied/satisfied/ Signed-off-by: Geert Uytterhoeven Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hfi1/tid_rdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index bdf1c313e13f..8f564b0a3333 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -477,7 +477,7 @@ static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd, * Must hold the qp s_lock and the exp_lock. * * Return: - * false if either of the conditions below are statisfied: + * false if either of the conditions below are satisfied: * 1. The list is empty or * 2. The indicated qp is at the head of the list and the * HFI1_S_WAIT_TID_SPACE bit is set in qp->s_flags. -- cgit v1.2.3 From cecae747b6208d40d08f8336393345093536b124 Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Wed, 12 Jun 2019 15:20:13 +0300 Subject: RDMA/mlx5: Consider eswitch encap mode When flow steering is created, then the encap support should consider the eswitch encap mode. If the eswitch flow table (FDB) supports encap then it shouldn't be supported on NIC RX flow tables. Fixes: 4adda1122c490 ('RDMA/mlx5: Enable decap and packet reformat on flow tables') Signed-off-by: Maor Gottlieb Reviewed-by: Petr Vorel Signed-off-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/mlx5/main.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 99eb4a8b0b0d..6a69be02ed0d 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -3252,11 +3253,14 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, int max_table_size; int num_entries; int num_groups; + bool esw_encap; u32 flags = 0; int priority; max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != + DEVLINK_ESWITCH_ENCAP_MODE_NONE; if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { enum mlx5_flow_namespace_type fn_type; @@ -3269,10 +3273,10 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, if (ft_type == MLX5_IB_FT_RX) { fn_type = MLX5_FLOW_NAMESPACE_BYPASS; prio = &dev->flow_db->prios[priority]; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, reformat_l3_tunnel_to_l2)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; @@ -3282,7 +3286,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, log_max_ft_size)); fn_type = MLX5_FLOW_NAMESPACE_EGRESS; prio = &dev->flow_db->egress_prios[priority]; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } @@ -3892,6 +3896,7 @@ _get_flow_table(struct mlx5_ib_dev *dev, struct mlx5_flow_namespace *ns = NULL; struct mlx5_ib_flow_prio *prio = NULL; int max_table_size = 0; + bool esw_encap; u32 flags = 0; int priority; @@ -3900,18 +3905,21 @@ _get_flow_table(struct mlx5_ib_dev *dev, else priority = ib_prio_to_core_prio(fs_matcher->priority, false); + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != + DEVLINK_ESWITCH_ENCAP_MODE_NONE; if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, - reformat_l3_tunnel_to_l2)) + reformat_l3_tunnel_to_l2) && + !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { max_table_size = BIT( MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); - if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { max_table_size = BIT( -- cgit v1.2.3 From 09d985bea99da4991076a53e104ec213ac518adb Mon Sep 17 00:00:00 2001 From: Maor Gottlieb Date: Wed, 12 Jun 2019 15:20:14 +0300 Subject: RDMA/mlx5: Enable decap and packet reformat on FDB If FDB flow tables support decap operation, enable it on creation, This allows to perform decapsulation of tunnelled packets by steering rules. If FDB flow tables support reformat operation, enable it on creation as well. Signed-off-by: Maor Gottlieb Reviewed-by: Petr Vorel Signed-off-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/mlx5/main.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 6a69be02ed0d..3b1985215cb9 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3924,6 +3924,11 @@ _get_flow_table(struct mlx5_ib_dev *dev, } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { max_table_size = BIT( MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && + esw_encap) + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; priority = FDB_BYPASS_PATH; } -- cgit v1.2.3 From 38389eaa4db192648916464b60f6086d6bbaa6de Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Sat, 8 Jun 2019 14:46:08 +0800 Subject: RDMA/hns: Add mtr support for mixed multihop addressing Currently, the MTT(memory translate table) design required a buffer space must has the same hopnum, but the hip08 hw can support mixed hopnum config in a buffer space. This patch adds the MTR(memory translate region) design for supporting mixed multihop. Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hns/hns_roce_device.h | 36 +++ drivers/infiniband/hw/hns/hns_roce_hem.c | 460 ++++++++++++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_hem.h | 14 + drivers/infiniband/hw/hns/hns_roce_mr.c | 118 +++++++ 4 files changed, 628 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 303ea7c614a8..ec2ed5cac3bc 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -341,6 +341,29 @@ struct hns_roce_mtt { enum hns_roce_mtt_type mtt_type; }; +struct hns_roce_buf_region { + int offset; /* page offset */ + u32 count; /* page count*/ + int hopnum; /* addressing hop num */ +}; + +#define HNS_ROCE_MAX_BT_REGION 3 +#define HNS_ROCE_MAX_BT_LEVEL 3 +struct hns_roce_hem_list { + struct list_head root_bt; + /* link all bt dma mem by hop config */ + struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL]; + struct list_head btm_bt; /* link all bottom bt in @mid_bt */ + dma_addr_t root_ba; /* pointer to the root ba table */ + int bt_pg_shift; +}; + +/* memory translate region */ +struct hns_roce_mtr { + struct hns_roce_hem_list hem_list; + int buf_pg_shift; +}; + struct hns_roce_mw { struct ib_mw ibmw; u32 pdn; @@ -1111,6 +1134,19 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct hns_roce_buf *buf); +void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift, + int buf_pg_shift); +int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t **bufs, struct hns_roce_buf_region *regions, + int region_cnt); +void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr); + +/* hns roce hw need current block and next block addr from mtt */ +#define MTT_MIN_COUNT 2 +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr); + int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev); diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 8490a86c3ef0..d145e3ed21d4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -1157,3 +1157,463 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) &hr_dev->mr_table.mtt_cqe_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); } + +struct roce_hem_item { + struct list_head list; /* link all hems in the same bt level */ + struct list_head sibling; /* link all hems in last hop for mtt */ + void *addr; + dma_addr_t dma_addr; + size_t count; /* max ba numbers */ + int start; /* start buf offset in this hem */ + int end; /* end buf offset in this hem */ +}; + +static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, + int start, int end, + int count, bool exist_bt, + int bt_level) +{ + struct roce_hem_item *hem; + + hem = kzalloc(sizeof(*hem), GFP_KERNEL); + if (!hem) + return NULL; + + if (exist_bt) { + hem->addr = dma_alloc_coherent(hr_dev->dev, + count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); + if (!hem->addr) { + kfree(hem); + return NULL; + } + } + + hem->count = count; + hem->start = start; + hem->end = end; + INIT_LIST_HEAD(&hem->list); + INIT_LIST_HEAD(&hem->sibling); + + return hem; +} + +static void hem_list_free_item(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, bool exist_bt) +{ + if (exist_bt) + dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, + hem->addr, hem->dma_addr); + kfree(hem); +} + +static void hem_list_free_all(struct hns_roce_dev *hr_dev, + struct list_head *head, bool exist_bt) +{ + struct roce_hem_item *hem, *temp_hem; + + list_for_each_entry_safe(hem, temp_hem, head, list) { + list_del(&hem->list); + hem_list_free_item(hr_dev, hem, exist_bt); + } +} + +static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, + u64 table_addr) +{ + *(u64 *)(base_addr) = table_addr; +} + +/* assign L0 table address to hem from root bt */ +static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, void *cpu_addr, + u64 phy_addr) +{ + hem->addr = cpu_addr; + hem->dma_addr = (dma_addr_t)phy_addr; +} + +static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, + int offset) +{ + return (hem->start <= offset && offset <= hem->end); +} + +static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) +{ + struct roce_hem_item *hem, *temp_hem; + struct roce_hem_item *found = NULL; + + list_for_each_entry_safe(hem, temp_hem, ba_list, list) { + if (hem_list_page_is_in_range(hem, page_offset)) { + found = hem; + break; + } + } + + return found; +} + +static bool hem_list_is_bottom_bt(int hopnum, int bt_level) +{ + /* + * hopnum base address table levels + * 0 L0(buf) + * 1 L0 -> buf + * 2 L0 -> L1 -> buf + * 3 L0 -> L1 -> L2 -> buf + */ + return bt_level >= (hopnum ? hopnum - 1 : hopnum); +} + +/** + * calc base address entries num + * @hopnum: num of mutihop addressing + * @bt_level: base address table level + * @unit: ba entries per bt page + */ +static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) +{ + u32 step; + int max; + int i; + + if (hopnum <= bt_level) + return 0; + /* + * hopnum bt_level range + * 1 0 unit + * ------------ + * 2 0 unit * unit + * 2 1 unit + * ------------ + * 3 0 unit * unit * unit + * 3 1 unit * unit + * 3 2 unit + */ + step = 1; + max = hopnum - bt_level; + for (i = 0; i < max; i++) + step = step * unit; + + return step; +} + +/** + * calc the root ba entries which could cover all regions + * @regions: buf region array + * @region_cnt: array size of @regions + * @unit: ba entries per bt page + */ +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit) +{ + struct hns_roce_buf_region *r; + int total = 0; + int step; + int i; + + for (i = 0; i < region_cnt; i++) { + r = (struct hns_roce_buf_region *)®ions[i]; + if (r->hopnum > 1) { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step > 0) + total += (r->count + step - 1) / step; + } else { + total += r->count; + } + } + + return total; +} + +static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, + const struct hns_roce_buf_region *r, int unit, + int offset, struct list_head *mid_bt, + struct list_head *btm_bt) +{ + struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; + struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; + struct roce_hem_item *cur, *pre; + const int hopnum = r->hopnum; + int start_aligned; + int distance; + int ret = 0; + int max_ofs; + int level; + u32 step; + int end; + + if (hopnum <= 1) + return 0; + + if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { + dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); + return -EINVAL; + } + + if (offset < r->offset) { + dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", + offset, r->offset); + return -EINVAL; + } + + distance = offset - r->offset; + max_ofs = r->offset + r->count - 1; + for (level = 0; level < hopnum; level++) + INIT_LIST_HEAD(&temp_list[level]); + + /* config L1 bt to last bt and link them to corresponding parent */ + for (level = 1; level < hopnum; level++) { + cur = hem_list_search_item(&mid_bt[level], offset); + if (cur) { + hem_ptrs[level] = cur; + continue; + } + + step = hem_list_calc_ba_range(hopnum, level, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + + start_aligned = (distance / step) * step + r->offset; + end = min_t(int, start_aligned + step - 1, max_ofs); + cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, + true, level); + if (!cur) { + ret = -ENOMEM; + goto err_exit; + } + hem_ptrs[level] = cur; + list_add(&cur->list, &temp_list[level]); + if (hem_list_is_bottom_bt(hopnum, level)) + list_add(&cur->sibling, &temp_list[0]); + + /* link bt to parent bt */ + if (level > 1) { + pre = hem_ptrs[level - 1]; + step = (cur->start - pre->start) / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, pre->addr + step, + cur->dma_addr); + } + } + + list_splice(&temp_list[0], btm_bt); + for (level = 1; level < hopnum; level++) + list_splice(&temp_list[level], &mid_bt[level]); + + return 0; + +err_exit: + for (level = 1; level < hopnum; level++) + hem_list_free_all(hr_dev, &temp_list[level], true); + + return ret; +} + +static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, int unit, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct roce_hem_item *hem, *temp_hem, *root_hem; + struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; + const struct hns_roce_buf_region *r; + struct list_head temp_root; + struct list_head temp_btm; + void *cpu_base; + u64 phy_base; + int ret = 0; + int offset; + int total; + int step; + int i; + + r = ®ions[0]; + root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); + if (root_hem) + return 0; + + INIT_LIST_HEAD(&temp_root); + total = r->offset; + /* indicate to last region */ + r = ®ions[region_cnt - 1]; + root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1, + unit, true, 0); + if (!root_hem) + return -ENOMEM; + list_add(&root_hem->list, &temp_root); + + hem_list->root_ba = root_hem->dma_addr; + + INIT_LIST_HEAD(&temp_btm); + for (i = 0; i < region_cnt; i++) + INIT_LIST_HEAD(&temp_list[i]); + + total = 0; + for (i = 0; i < region_cnt && total < unit; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + /* all regions's mid[x][0] shared the root_bt's trunk */ + cpu_base = root_hem->addr + total * BA_BYTE_LEN; + phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; + + /* if hopnum is 0 or 1, cut a new fake hem from the root bt + * which's address share to all regions. + */ + if (hem_list_is_bottom_bt(r->hopnum, 0)) { + hem = hem_list_alloc_item(hr_dev, r->offset, + r->offset + r->count - 1, + r->count, false, 0); + if (!hem) { + ret = -ENOMEM; + goto err_exit; + } + hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + list_add(&hem->list, &temp_list[i]); + list_add(&hem->sibling, &temp_btm); + total += r->count; + } else { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry_safe(hem, temp_hem, + &hem_list->mid_bt[i][1], list) { + offset = hem->start / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, cpu_base + offset, + hem->dma_addr); + total++; + } + } + } + + list_splice(&temp_btm, &hem_list->btm_bt); + list_splice(&temp_root, &hem_list->root_bt); + for (i = 0; i < region_cnt; i++) + list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); + + return 0; + +err_exit: + for (i = 0; i < region_cnt; i++) + hem_list_free_all(hr_dev, &temp_list[i], false); + + hem_list_free_all(hr_dev, &temp_root, true); + + return ret; +} + +/* construct the base address table and link them by address hop config */ +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + const struct hns_roce_buf_region *r; + int ofs, end; + int ret = 0; + int unit; + int i; + + if (region_cnt > HNS_ROCE_MAX_BT_REGION) { + dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", + region_cnt); + return -EINVAL; + } + + unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN; + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + end = r->offset + r->count; + for (ofs = r->offset; ofs < end; ofs += unit) { + ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, + hem_list->mid_bt[i], + &hem_list->btm_bt); + if (ret) { + dev_err(hr_dev->dev, + "alloc hem trunk fail ret=%d!\n", ret); + goto err_alloc; + } + } + } + + ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, + region_cnt); + if (ret) + dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); + else + return 0; + +err_alloc: + hns_roce_hem_list_release(hr_dev, hem_list); + + return ret; +} + +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list) +{ + int i, j; + + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], + j != 0); + + hem_list_free_all(hr_dev, &hem_list->root_bt, true); + INIT_LIST_HEAD(&hem_list->btm_bt); + hem_list->root_ba = 0; +} + +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, + int bt_page_order) +{ + int i, j; + + INIT_LIST_HEAD(&hem_list->root_bt); + INIT_LIST_HEAD(&hem_list->btm_bt); + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); + + hem_list->bt_pg_shift = bt_page_order; +} + +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr) +{ + struct list_head *head = &hem_list->btm_bt; + struct roce_hem_item *hem, *temp_hem; + void *cpu_base = NULL; + u64 phy_base = 0; + int nr = 0; + + list_for_each_entry_safe(hem, temp_hem, head, sibling) { + if (hem_list_page_is_in_range(hem, offset)) { + nr = offset - hem->start; + cpu_base = hem->addr + nr * BA_BYTE_LEN; + phy_base = hem->dma_addr + nr * BA_BYTE_LEN; + nr = hem->end + 1 - offset; + break; + } + } + + if (mtt_cnt) + *mtt_cnt = nr; + + if (phy_addr) + *phy_addr = phy_base; + + return cpu_base; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index d9d668992e49..e865fc8a18a7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -133,6 +133,20 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, struct hns_roce_hem_mhop *mhop); bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type); +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, + int bt_page_order); +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit); +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt); +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list); +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr); + static inline void hns_roce_hem_first(struct hns_roce_hem *hem, struct hns_roce_hem_iter *iter) { diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 38ed4ac741b5..6db0dae18ab7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -1496,3 +1496,121 @@ int hns_roce_dealloc_mw(struct ib_mw *ibmw) return 0; } + +void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift, + int buf_pg_shift) +{ + hns_roce_hem_list_init(&mtr->hem_list, bt_pg_shift); + mtr->buf_pg_shift = buf_pg_shift; +} + +void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr) +{ + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); +} +EXPORT_SYMBOL_GPL(hns_roce_mtr_cleanup); + +static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr, dma_addr_t *bufs, + struct hns_roce_buf_region *r) +{ + int offset; + int count; + int npage; + u64 *mtts; + int end; + int i; + + offset = r->offset; + end = offset + r->count; + npage = 0; + while (offset < end) { + mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + offset, &count, NULL); + if (!mtts) + return -ENOBUFS; + + /* Save page addr, low 12 bits : 0 */ + for (i = 0; i < count; i++) { + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) + mtts[i] = cpu_to_le64(bufs[npage] >> + PAGE_ADDR_SHIFT); + else + mtts[i] = cpu_to_le64(bufs[npage]); + + npage++; + } + offset += count; + } + + return 0; +} + +int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t **bufs, struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct hns_roce_buf_region *r; + int ret; + int i; + + ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions, + region_cnt); + if (ret) + return ret; + + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r); + if (ret) { + dev_err(hr_dev->dev, + "write mtr[%d/%d] err %d,offset=%d.\n", + i, region_cnt, ret, r->offset); + goto err_write; + } + } + + return 0; + +err_write: + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); + + return ret; +} + +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) +{ + u64 *mtts = mtt_buf; + int mtt_count; + int total = 0; + u64 *addr; + int npage; + int left; + + if (mtts == NULL || mtt_max < 1) + goto done; + + left = mtt_max; + while (left > 0) { + mtt_count = 0; + addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + offset + total, + &mtt_count, NULL); + if (!addr || !mtt_count) + goto done; + + npage = min(mtt_count, left); + memcpy(&mtts[total], addr, BA_BYTE_LEN * npage); + left -= npage; + total += npage; + } + +done: + if (base_addr) + *base_addr = mtr->hem_list.root_ba; + + return total; +} +EXPORT_SYMBOL_GPL(hns_roce_mtr_find); -- cgit v1.2.3 From 2ac0bc5e725e84d56b7fc8b280b2dbc9f2f10737 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Sat, 8 Jun 2019 14:46:09 +0800 Subject: RDMA/hns: Add a group interfaces for optimizing buffers getting flow Currently, the code for getting umem and kmem buffers exist many files, this patch adds a group interfaces to simplify the buffers getting flow. Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hns/hns_roce_alloc.c | 99 +++++++++++++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_device.h | 12 ++++ 2 files changed, 111 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index dac058d3df53..14fcc359599c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -34,6 +34,7 @@ #include #include #include "hns_roce_device.h" +#include int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj) { @@ -238,6 +239,104 @@ err_free: return -ENOMEM; } +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf) +{ + int i, end; + int total; + + end = start + buf_cnt; + if (end > buf->npages) { + dev_err(hr_dev->dev, + "invalid kmem region,offset %d,buf_cnt %d,total %d!\n", + start, buf_cnt, buf->npages); + return -EINVAL; + } + + total = 0; + for (i = start; i < end; i++) + if (buf->nbufs == 1) + bufs[total++] = buf->direct.map + + (i << buf->page_shift); + else + bufs[total++] = buf->page_list[i].map; + + return total; +} + +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + int page_shift) +{ + struct ib_block_iter biter; + int total = 0; + int idx = 0; + u64 addr; + + if (page_shift < PAGE_SHIFT) { + dev_err(hr_dev->dev, "invalid page shift %d!\n", page_shift); + return -EINVAL; + } + + /* convert system page cnt to hw page cnt */ + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + 1 << page_shift) { + addr = rdma_block_iter_dma_address(&biter); + if (idx >= start) { + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; + } + idx++; + } + +done: + return total; +} + +void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum, + int offset, int buf_cnt) +{ + if (hopnum == HNS_ROCE_HOP_NUM_0) + region->hopnum = 0; + else + region->hopnum = hopnum; + + region->offset = offset; + region->count = buf_cnt; +} + +void hns_roce_free_buf_list(dma_addr_t **bufs, int region_cnt) +{ + int i; + + for (i = 0; i < region_cnt; i++) { + kfree(bufs[i]); + bufs[i] = NULL; + } +} + +int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions, + dma_addr_t **bufs, int region_cnt) +{ + struct hns_roce_buf_region *r; + int i; + + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + bufs[i] = kcalloc(r->count, sizeof(dma_addr_t), GFP_KERNEL); + if (!bufs[i]) + goto err_alloc; + } + + return 0; + +err_alloc: + hns_roce_free_buf_list(bufs, i); + + return -ENOMEM; +} + void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) { if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index ec2ed5cac3bc..4974c42a77de 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1211,6 +1211,18 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct ib_umem *umem); +void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum, + int offset, int buf_cnt); +int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions, + dma_addr_t **bufs, int count); +void hns_roce_free_buf_list(dma_addr_t **bufs, int count); + +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf); +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + int page_shift); + int hns_roce_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); -- cgit v1.2.3 From 8d18ad83f19b7fb67485f977a51408287e3f801f Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Sat, 8 Jun 2019 14:46:10 +0800 Subject: RDMA/hns: Fix bug when wqe num is larger than 16K hip08 can support up to 32768 wqes in one qp. currently if the wqe num is larger than 16384, the driver will lead a calltrace as follows. [21361.393725] Call trace: [21361.398605] hns_roce_v2_modify_qp+0xbcc/0x1360 [hns_roce_hw_v2] [21361.410627] hns_roce_modify_qp+0x1d8/0x2f8 [hns_roce] [21361.420906] _ib_modify_qp+0x70/0x118 [21361.428222] ib_modify_qp+0x14/0x1c [21361.435193] rt_ktest_modify_qp+0xb8/0x650 [rdma_test] [21361.445472] exec_modify_qp_cmd+0x110/0x4d8 [rdma_test] [21361.455924] rt_ktest_dispatch_cmd_3+0xa94/0x2edc [rdma_test] [21361.467422] rt_ktest_dispatch_cmd_2+0x9c/0x108 [rdma_test] [21361.478570] rt_ktest_dispatch_cmd+0x138/0x904 [rdma_test] [21361.489545] rt_ktest_dev_write+0x328/0x4b0 [rdma_test] [21361.499998] __vfs_write+0x38/0x15c [21361.506966] vfs_write+0xa8/0x1a0 [21361.513586] ksys_write+0x50/0xb0 [21361.520206] sys_write+0xc/0x14 [21361.526479] el0_svc_naked+0x30/0x34 [21361.533622] Code: 1ac10841 d37d7c22 0b000021 d37df021 (f86268c0) [21361.545815] ---[ end trace e2a1feb2c3d7f13c ]--- When the wqe num is larger than 16384, hns_roce_table_find will return an invalid mtt, this will lead an kernel paging requet error if the driver try to access it. It's the mtt design defect which can't support up to the max wqe num of hip08. This patch fixs it by replacing mtt with mtr for wqe. Fixes: 926a01dc000d ("RDMA/hns: Add QP operations support for hip08 SoC") Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hns/hns_roce_device.h | 11 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 118 +++++++++++------ drivers/infiniband/hw/hns/hns_roce_qp.c | 189 +++++++++++++++++++++------- 3 files changed, 235 insertions(+), 83 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 4974c42a77de..a548b28aab63 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -660,6 +660,14 @@ struct hns_roce_qp { struct ib_umem *umem; struct hns_roce_mtt mtt; + struct hns_roce_mtr mtr; + + /* this define must less than HNS_ROCE_MAX_BT_REGION */ +#define HNS_ROCE_WQE_REGION_MAX 3 + struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX]; + int region_cnt; + int wqe_bt_pg_shift; + u32 buff_size; struct mutex mutex; u8 port; @@ -870,6 +878,9 @@ struct hns_roce_caps { u32 mtt_ba_pg_sz; u32 mtt_buf_pg_sz; u32 mtt_hop_num; + u32 wqe_sq_hop_num; + u32 wqe_sge_hop_num; + u32 wqe_rq_hop_num; u32 sccc_ba_pg_sz; u32 sccc_buf_pg_sz; u32 sccc_hop_num; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 2d27dc91a823..e7024b3e171a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1574,6 +1574,9 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->mtt_ba_pg_sz = 0; caps->mtt_buf_pg_sz = 0; caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; + caps->wqe_sq_hop_num = 2; + caps->wqe_sge_hop_num = 1; + caps->wqe_rq_hop_num = 2; caps->cqe_ba_pg_sz = 0; caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; @@ -3021,7 +3024,6 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, } static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, - struct hns_roce_mtt *mtt, enum ib_qp_state cur_state, enum ib_qp_state new_state, struct hns_roce_v2_qp_context *context, @@ -3517,6 +3519,31 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, } } +static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, int mtt_cnt, + u32 page_size) +{ + struct device *dev = hr_dev->dev; + + if (hr_qp->rq.wqe_cnt < 1) + return true; + + if (mtt_cnt < 1) { + dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n", + hr_qp->qpn); + return false; + } + + if (mtt_cnt < MTT_MIN_COUNT && + (hr_qp->rq.offset + page_size) < hr_qp->buff_size) { + dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n", + hr_qp->qpn); + return false; + } + + return true; +} + static int modify_qp_init_to_rtr(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, struct hns_roce_v2_qp_context *context, @@ -3526,25 +3553,27 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct device *dev = hr_dev->dev; + u64 mtts[MTT_MIN_COUNT] = { 0 }; dma_addr_t dma_handle_3; dma_addr_t dma_handle_2; - dma_addr_t dma_handle; + u64 wqe_sge_ba; u32 page_size; u8 port_num; u64 *mtts_3; u64 *mtts_2; - u64 *mtts; + int count; u8 *dmac; u8 *smac; int port; /* Search qp buf's mtts */ - mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, - hr_qp->mtt.first_seg, &dma_handle); - if (!mtts) { - dev_err(dev, "qp buf pa find failed\n"); - return -EINVAL; - } + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, + hr_qp->rq.offset / page_size, mtts, + MTT_MIN_COUNT, &wqe_sge_ba); + if (!ibqp->srq) + if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) + return -EINVAL; /* Search IRRL's mtts */ mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, @@ -3568,7 +3597,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, } dmac = (u8 *)attr->ah_attr.roce.dmac; - context->wqe_sge_ba = (u32)(dma_handle >> 3); + context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3); qpc_mask->wqe_sge_ba = 0; /* @@ -3578,22 +3607,23 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, * 0 at the same time, else set them to 0x1. */ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, - V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3)); + V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3)); roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, V2_QPC_BYTE_12_SQ_HOP_NUM_S, - hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? - 0 : hr_dev->caps.mtt_hop_num); + hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.wqe_sq_hop_num); roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGE_HOP_NUM_M, V2_QPC_BYTE_20_SGE_HOP_NUM_S, - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? - hr_dev->caps.mtt_hop_num : 0); + ((ibqp->qp_type == IB_QPT_GSI) || + hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? + hr_dev->caps.wqe_sge_hop_num : 0); roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGE_HOP_NUM_M, V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); @@ -3601,8 +3631,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQ_HOP_NUM_M, V2_QPC_BYTE_20_RQ_HOP_NUM_S, - hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? - 0 : hr_dev->caps.mtt_hop_num); + hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.wqe_rq_hop_num); roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQ_HOP_NUM_M, V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); @@ -3610,7 +3640,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, - hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET); + hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET); roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); @@ -3623,29 +3653,24 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); - page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size] - >> PAGE_ADDR_SHIFT); + context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); qpc_mask->rq_cur_blk_addr = 0; roce_set_field(context->byte_92_srq_info, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, - mtts[hr_qp->rq.offset / page_size] - >> (32 + PAGE_ADDR_SHIFT)); + mtts[0] >> (32 + PAGE_ADDR_SHIFT)); roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); - context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1] - >> PAGE_ADDR_SHIFT); + context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT); qpc_mask->rq_nxt_blk_addr = 0; roce_set_field(context->byte_104_rq_sge, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, - mtts[hr_qp->rq.offset / page_size + 1] - >> (32 + PAGE_ADDR_SHIFT)); + mtts[1] >> (32 + PAGE_ADDR_SHIFT)); roce_set_field(qpc_mask->byte_104_rq_sge, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); @@ -3773,18 +3798,30 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct device *dev = hr_dev->dev; - dma_addr_t dma_handle; + u64 sge_cur_blk = 0; + u64 sq_cur_blk = 0; u32 page_size; - u64 *mtts; + int count; /* Search qp buf's mtts */ - mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, - hr_qp->mtt.first_seg, &dma_handle); - if (!mtts) { - dev_err(dev, "qp buf pa find failed\n"); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); + if (count < 1) { + dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn); return -EINVAL; } + if (hr_qp->sge.offset) { + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, + hr_qp->sge.offset / page_size, + &sge_cur_blk, 1, NULL); + if (count < 1) { + dev_err(dev, "qp(0x%lx) sge pa find failed\n", + hr_qp->qpn); + return -EINVAL; + } + } + /* Not support alternate path and path migration */ if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_PATH_MIG_STATE)) { @@ -3798,38 +3835,37 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, * we should set all bits of the relevant fields in context mask to * 0 at the same time, else set them to 0x1. */ - context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); roce_set_field(context->byte_168_irrl_idx, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, - mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); qpc_mask->sq_cur_blk_addr = 0; roce_set_field(qpc_mask->byte_168_irrl_idx, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); - page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? - ((u32)(mtts[hr_qp->sge.offset / page_size] >> + ((u32)(sge_cur_blk >> PAGE_ADDR_SHIFT)) : 0; roce_set_field(context->byte_184_irrl_idx, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? - (mtts[hr_qp->sge.offset / page_size] >> + (sge_cur_blk >> (32 + PAGE_ADDR_SHIFT)) : 0); qpc_mask->sq_cur_sge_blk_addr = 0; roce_set_field(qpc_mask->byte_184_irrl_idx, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); - context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); roce_set_field(context->byte_232_irrl_sge, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, - mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); qpc_mask->rx_sq_cur_blk_addr = 0; roce_set_field(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, @@ -4230,7 +4266,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, V2_QPC_BYTE_60_QP_ST_S, 0); /* SW pass context to HW */ - ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state, + ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, context, hr_qp); if (ret) { dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 8db2817a249e..99ec5d43b99b 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -422,6 +422,91 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, return 0; } +static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_buf_region *regions, + int region_max, int page_shift) +{ + int page_size = 1 << page_shift; + bool is_extend_sge; + int region_cnt = 0; + int buf_size; + int buf_cnt; + + if (hr_qp->buff_size < 1 || region_max < 1) + return region_cnt; + + if (hr_qp->sge.sge_cnt > 0) + is_extend_sge = true; + else + is_extend_sge = false; + + /* sq region */ + if (is_extend_sge) + buf_size = hr_qp->sge.offset - hr_qp->sq.offset; + else + buf_size = hr_qp->rq.offset - hr_qp->sq.offset; + + if (buf_size > 0 && region_cnt < region_max) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_sq_hop_num, + hr_qp->sq.offset / page_size, + buf_cnt); + region_cnt++; + } + + /* sge region */ + if (is_extend_sge) { + buf_size = hr_qp->rq.offset - hr_qp->sge.offset; + if (buf_size > 0 && region_cnt < region_max) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_sge_hop_num, + hr_qp->sge.offset / page_size, + buf_cnt); + region_cnt++; + } + } + + /* rq region */ + buf_size = hr_qp->buff_size - hr_qp->rq.offset; + if (buf_size > 0) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_rq_hop_num, + hr_qp->rq.offset / page_size, + buf_cnt); + region_cnt++; + } + + return region_cnt; +} + +static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, + struct hns_roce_buf_region *regions, + int region_cnt) +{ + int bt_pg_shift; + int ba_num; + int ret; + + bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; + + /* all root ba entries must in one bt page */ + do { + ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN; + ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt, + ba_num); + if (ret <= ba_num) + break; + + bt_pg_shift++; + } while (ret > ba_num); + + return bt_pg_shift - PAGE_SHIFT; +} + static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) @@ -534,15 +619,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_udata *udata, unsigned long sqpn, struct hns_roce_qp *hr_qp) { + dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { 0 }; struct device *dev = hr_dev->dev; struct hns_roce_ib_create_qp ucmd; struct hns_roce_ib_create_qp_resp resp = {}; struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( udata, struct hns_roce_ucontext, ibucontext); + struct hns_roce_buf_region *r; unsigned long qpn = 0; - int ret = 0; u32 page_shift; - u32 npages; + int buf_count; + int ret; int i; mutex_init(&hr_qp->mutex); @@ -596,6 +683,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, init_attr->cap.max_recv_sge]; } + page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; if (udata) { if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { dev_err(dev, "ib_copy_from_udata error for create qp\n"); @@ -617,32 +705,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = PTR_ERR(hr_qp->umem); goto err_rq_sge_list; } - - hr_qp->mtt.mtt_type = MTT_TYPE_WQE; - page_shift = PAGE_SHIFT; - if (hr_dev->caps.mtt_buf_pg_sz) { - npages = (ib_umem_page_count(hr_qp->umem) + - (1 << hr_dev->caps.mtt_buf_pg_sz) - 1) / - (1 << hr_dev->caps.mtt_buf_pg_sz); - page_shift += hr_dev->caps.mtt_buf_pg_sz; - ret = hns_roce_mtt_init(hr_dev, npages, - page_shift, - &hr_qp->mtt); - } else { - ret = hns_roce_mtt_init(hr_dev, - ib_umem_page_count(hr_qp->umem), - page_shift, &hr_qp->mtt); - } + hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, + hr_qp->regions, ARRAY_SIZE(hr_qp->regions), + page_shift); + ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, + hr_qp->region_cnt); if (ret) { - dev_err(dev, "hns_roce_mtt_init error for create qp\n"); - goto err_buf; + dev_err(dev, "alloc buf_list error for create qp\n"); + goto err_alloc_list; } - ret = hns_roce_ib_umem_write_mtt(hr_dev, &hr_qp->mtt, - hr_qp->umem); - if (ret) { - dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n"); - goto err_mtt; + for (i = 0; i < hr_qp->region_cnt; i++) { + r = &hr_qp->regions[i]; + buf_count = hns_roce_get_umem_bufs(hr_dev, + buf_list[i], r->count, r->offset, + hr_qp->umem, page_shift); + if (buf_count != r->count) { + dev_err(dev, + "get umem buf err, expect %d,ret %d.\n", + r->count, buf_count); + ret = -ENOBUFS; + goto err_get_bufs; + } } if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && @@ -653,7 +737,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, &hr_qp->sdb); if (ret) { dev_err(dev, "sq record doorbell map failed!\n"); - goto err_mtt; + goto err_get_bufs; } /* indicate kernel supports sq record db */ @@ -715,7 +799,6 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } /* Allocate QP buf */ - page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, (1 << page_shift) * 2, &hr_qp->hr_buf, page_shift)) { @@ -723,21 +806,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = -ENOMEM; goto err_db; } - - hr_qp->mtt.mtt_type = MTT_TYPE_WQE; - /* Write MTT */ - ret = hns_roce_mtt_init(hr_dev, hr_qp->hr_buf.npages, - hr_qp->hr_buf.page_shift, &hr_qp->mtt); + hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, + hr_qp->regions, ARRAY_SIZE(hr_qp->regions), + page_shift); + ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, + hr_qp->region_cnt); if (ret) { - dev_err(dev, "hns_roce_mtt_init error for kernel create qp\n"); - goto err_buf; + dev_err(dev, "alloc buf_list error for create qp!\n"); + goto err_alloc_list; } - ret = hns_roce_buf_write_mtt(hr_dev, &hr_qp->mtt, - &hr_qp->hr_buf); - if (ret) { - dev_err(dev, "hns_roce_buf_write_mtt error for kernel create qp\n"); - goto err_mtt; + for (i = 0; i < hr_qp->region_cnt; i++) { + r = &hr_qp->regions[i]; + buf_count = hns_roce_get_kmem_bufs(hr_dev, + buf_list[i], r->count, r->offset, + &hr_qp->hr_buf); + if (buf_count != r->count) { + dev_err(dev, + "get kmem buf err, expect %d,ret %d.\n", + r->count, buf_count); + ret = -ENOBUFS; + goto err_get_bufs; + } } hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), @@ -761,6 +851,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } } + hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, + hr_qp->region_cnt); + hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, + page_shift); + ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, + hr_qp->regions, hr_qp->region_cnt); + if (ret) { + dev_err(dev, "mtr attatch error for create qp\n"); + goto err_mtr; + } + if (init_attr->qp_type == IB_QPT_GSI && hr_dev->hw_rev == HNS_ROCE_HW_VER1) { /* In v1 engine, GSI QP context in RoCE engine's register */ @@ -796,6 +897,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } hr_qp->event = hns_roce_ib_qp_event; + hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); return 0; @@ -810,6 +912,9 @@ err_qpn: if (!sqpn) hns_roce_release_range_qp(hr_dev, qpn, 1); +err_mtr: + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); + err_wrid: if (udata) { if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && @@ -829,10 +934,10 @@ err_sq_dbmap: hns_roce_qp_has_sq(init_attr)) hns_roce_db_unmap_user(uctx, &hr_qp->sdb); -err_mtt: - hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); +err_get_bufs: + hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); -err_buf: +err_alloc_list: if (hr_qp->umem) ib_umem_release(hr_qp->umem); else -- cgit v1.2.3 From a49b1dc7ae447d7085360cd587fc1c8b9ec6c871 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Wed, 12 Jun 2019 15:27:41 +0300 Subject: RDMA: Convert destroy_wq to be void All callers of destroy WQ are always success and there is no need to check their return value, so convert destroy_wq to be void. Signed-off-by: Leon Romanovsky Reviewed-by: Yuval Shaia Signed-off-by: Doug Ledford --- drivers/infiniband/hw/mlx4/mlx4_ib.h | 2 +- drivers/infiniband/hw/mlx4/qp.c | 4 +--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- drivers/infiniband/hw/mlx5/qp.c | 4 +--- 4 files changed, 4 insertions(+), 8 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 81b3d85e5167..eb53bb4c0c91 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -906,7 +906,7 @@ void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port); struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +void mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 5221c0794d1d..520364defa28 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -4248,7 +4248,7 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr, return err; } -int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) +void mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibwq->device); struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); @@ -4259,8 +4259,6 @@ int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata); kfree(qp); - - return 0; } struct ib_rwq_ind_table diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 35e2c8f5ae78..82cfe86087b6 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1201,7 +1201,7 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index f6623c77443a..ae847709b3d3 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -6047,7 +6047,7 @@ err: return ERR_PTR(err); } -int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) +void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(wq->device); struct mlx5_ib_rwq *rwq = to_mrwq(wq); @@ -6055,8 +6055,6 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp); destroy_user_rq(dev, wq->pd, rwq, udata); kfree(rwq); - - return 0; } struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, -- cgit v1.2.3 From 89a6da3cb8f30ee0aeca924d84bef688f22f883e Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Fri, 14 Jun 2019 22:56:03 +0800 Subject: RDMA/hns: reset function when removing module During removing the driver, we needs to notify the roce engine to stop working immediately,and symmetrically recycle the hardware resources requested during initialization. The hardware provides a command called function clear that can package these operations,so that the driver can only focus on releasing resources that applied from the operating system. This patch implements the call of this command. Signed-off-by: Lang Cheng Signed-off-by: Lijun Ou Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 44 ++++++++++++++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 17 ++++++++++++ 2 files changed, 61 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index e7024b3e171a..5c8551b54444 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1130,6 +1130,47 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) return 0; } +static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) +{ + bool fclr_write_fail_flag = false; + struct hns_roce_func_clear *resp; + struct hns_roce_cmq_desc desc; + unsigned long end; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); + resp = (struct hns_roce_func_clear *)desc.data; + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + fclr_write_fail_flag = true; + dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", + ret); + return; + } + + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); + end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; + while (end) { + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); + end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, + true); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + continue; + + if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { + hr_dev->is_reset = true; + return; + } + } + + dev_err(hr_dev->dev, "Func clear fail.\n"); +} + static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) { struct hns_roce_query_fw_info *resp; @@ -1894,6 +1935,9 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) { struct hns_roce_v2_priv *priv = hr_dev->priv; + if (hr_dev->pci_dev->revision == 0x21) + hns_roce_function_clear(hr_dev); + hns_roce_free_link_table(hr_dev, &priv->tpq); hns_roce_free_link_table(hr_dev, &priv->tsq); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index bce21fd2ebb6..478f5a5b7aa1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -241,6 +241,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_POST_MB = 0x8504, HNS_ROCE_OPC_QUERY_MB_ST = 0x8505, HNS_ROCE_OPC_CFG_BT_ATTR = 0x8506, + HNS_ROCE_OPC_FUNC_CLEAR = 0x8508, HNS_ROCE_OPC_CLR_SCCC = 0x8509, HNS_ROCE_OPC_QUERY_SCCC = 0x850a, HNS_ROCE_OPC_RESET_SCCC = 0x850b, @@ -1230,6 +1231,22 @@ struct hns_roce_query_fw_info { __le32 rsv[5]; }; +struct hns_roce_func_clear { + __le32 rst_funcid_en; + __le32 func_done; + __le32 rsv[4]; +}; + +#define FUNC_CLEAR_RST_FUN_DONE_S 0 +/* Each physical function manages up to 248 virtual functionsï¼› + * it takes up to 100ms for each function to execute clearï¼› + * if an abnormal reset occurs, it is executed twice at most; + * so it takes up to 249 * 2 * 100ms. + */ +#define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100) +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL 40 +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 + struct hns_roce_cfg_llm_a { __le32 base_addr_l; __le32 base_addr_h; -- cgit v1.2.3 From 836a0fbb3e76f704ad65ddfb57f00725245e509b Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 16 Jun 2019 15:05:20 +0300 Subject: RDMA: Check umem pointer validity prior to release Update ib_umem_release() to behave similarly to kfree() and allow submitting NULL pointer as safe input to this function. Fixes: a52c8e2469c3 ("RDMA: Clean destroy CQ in drivers do not return errors") Signed-off-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 29 +++++++++------------------- drivers/infiniband/hw/cxgb3/iwch_provider.c | 3 +-- drivers/infiniband/hw/cxgb4/mem.c | 3 +-- drivers/infiniband/hw/efa/efa_verbs.c | 2 +- drivers/infiniband/hw/hns/hns_roce_cq.c | 8 +++----- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 13 +++++-------- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- drivers/infiniband/hw/hns/hns_roce_mr.c | 4 +--- drivers/infiniband/hw/hns/hns_roce_qp.c | 5 ++--- drivers/infiniband/hw/hns/hns_roce_srq.c | 14 ++++++-------- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 3 +-- drivers/infiniband/hw/mlx4/cq.c | 14 +++++--------- drivers/infiniband/hw/mlx4/qp.c | 7 +++---- drivers/infiniband/hw/mlx4/srq.c | 7 +++---- drivers/infiniband/hw/mlx5/cq.c | 20 +++++-------------- drivers/infiniband/hw/mlx5/mr.c | 13 ++++++------- drivers/infiniband/hw/mlx5/qp.c | 9 +++------ drivers/infiniband/hw/mthca/mthca_provider.c | 3 +-- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 3 +-- drivers/infiniband/hw/qedr/verbs.c | 9 +++------ drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c | 6 ++---- drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c | 3 +-- drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c | 16 ++++----------- 23 files changed, 68 insertions(+), 128 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 44cc5f19df3b..a91653aabf38 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -805,10 +805,8 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) rdev->sqp_ah = NULL; } - if (!IS_ERR_OR_NULL(qp->rumem)) - ib_umem_release(qp->rumem); - if (!IS_ERR_OR_NULL(qp->sumem)) - ib_umem_release(qp->sumem); + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); mutex_lock(&rdev->qp_lock); list_del(&qp->list); @@ -1201,12 +1199,8 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, qp_destroy: bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); free_umem: - if (udata) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); fail: kfree(qp); return ERR_PTR(rc); @@ -1302,8 +1296,7 @@ void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (qplib_srq->cq) nq = qplib_srq->cq->nq; bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); - if (srq->umem) - ib_umem_release(srq->umem); + ib_umem_release(srq->umem); atomic_dec(&rdev->srq_count); if (nq) nq->budget--; @@ -1412,8 +1405,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, return 0; fail: - if (srq->umem) - ib_umem_release(srq->umem); + ib_umem_release(srq->umem); exit: return rc; } @@ -2528,8 +2520,7 @@ void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) nq = cq->qplib_cq.nq; bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); - if (!cq->umem) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); atomic_dec(&rdev->cq_count); nq->budget--; @@ -2632,8 +2623,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, return 0; c2fail: - if (udata) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); fail: kfree(cq->cql); return rc; @@ -3340,8 +3330,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) mr->npages = 0; mr->pages = NULL; } - if (!IS_ERR_OR_NULL(mr->ib_umem)) - ib_umem_release(mr->ib_umem); + ib_umem_release(mr->ib_umem); kfree(mr); atomic_dec(&rdev->mr_count); diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 810fa96af2e9..e775c1a1a450 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -346,8 +346,7 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) xa_erase_irq(&rhp->mrs, mmid); if (mhp->kva) kfree((void *) (unsigned long) mhp->kva); - if (mhp->umem) - ib_umem_release(mhp->umem); + ib_umem_release(mhp->umem); pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp); kfree(mhp); return 0; diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index fe3733c4026d..aa772ee0706f 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -808,8 +808,7 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) mhp->attr.pbl_size << 3); if (mhp->kva) kfree((void *) (unsigned long) mhp->kva); - if (mhp->umem) - ib_umem_release(mhp->umem); + ib_umem_release(mhp->umem); pr_debug("mmid 0x%x ptr %p\n", mmid, mhp); c4iw_put_wr_wait(mhp->wr_waitp); kfree(mhp); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index a9372c9e4b30..5e6e5eb65cff 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1513,8 +1513,8 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) err = efa_com_dereg_mr(&dev->edev, ¶ms); if (err) return err; - ib_umem_release(mr->umem); } + ib_umem_release(mr->umem); kfree(mr); diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 7e198c9ffbfe..6b4d8e50aabe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -423,9 +423,8 @@ err_dbmap: err_mtt: hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - if (udata) - ib_umem_release(hr_cq->umem); - else + ib_umem_release(hr_cq->umem); + if (!udata) hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, hr_cq->ib_cq.cqe); @@ -451,9 +450,8 @@ void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) hns_roce_free_cq(hr_dev, hr_cq); hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); + ib_umem_release(hr_cq->umem); if (udata) { - ib_umem_release(hr_cq->umem); - if (hr_cq->db_en == 1) hns_roce_db_unmap_user(rdma_udata_to_drv_context( udata, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index c899879da222..cb004190ccba 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -1163,8 +1163,7 @@ free_mr: hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, key_to_hw_index(mr->key), 0); - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); @@ -3641,9 +3640,8 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); - if (udata) - ib_umem_release(hr_qp->umem); - else { + ib_umem_release(hr_qp->umem); + if (!udata) { kfree(hr_qp->sq.wrid); kfree(hr_qp->rq.wrid); @@ -3694,9 +3692,8 @@ static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - if (ibcq->uobject) - ib_umem_release(hr_cq->umem); - else { + ib_umem_release(hr_cq->umem); + if (!udata) { /* Free the buff of stored cq */ cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz; hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf); diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 5c8551b54444..edd62b4dc0a0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4582,7 +4582,6 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1)) hns_roce_db_unmap_user(context, &hr_qp->rdb); - ib_umem_release(hr_qp->umem); } else { kfree(hr_qp->sq.wrid); kfree(hr_qp->rq.wrid); @@ -4590,6 +4589,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, if (hr_qp->rq.wqe_cnt) hns_roce_free_db(hr_dev, &hr_qp->rdb); } + ib_umem_release(hr_qp->umem); if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_qp->rq.wqe_cnt) { diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 6db0dae18ab7..adf075183dfb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -1298,9 +1298,7 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } else { hns_roce_mr_free(hr_dev, mr); - if (mr->umem) - ib_umem_release(mr->umem); - + ib_umem_release(mr->umem); kfree(mr); } diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 99ec5d43b99b..7e9db8236072 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -938,10 +938,9 @@ err_get_bufs: hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); err_alloc_list: - if (hr_qp->umem) - ib_umem_release(hr_qp->umem); - else + if (!hr_qp->umem) hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + ib_umem_release(hr_qp->umem); err_db: if (!udata && hns_roce_qp_has_rq(init_attr) && diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index c222f243953a..de645be8aa48 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -380,8 +380,7 @@ err_idx_buf: hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); err_idx_mtt: - if (udata) - ib_umem_release(srq->idx_que.umem); + ib_umem_release(srq->idx_que.umem); err_create_idx: hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, @@ -392,9 +391,8 @@ err_srq_mtt: hns_roce_mtt_cleanup(hr_dev, &srq->mtt); err_buf: - if (udata) - ib_umem_release(srq->umem); - else + ib_umem_release(srq->umem); + if (!udata) hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); return ret; @@ -408,15 +406,15 @@ void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) hns_roce_srq_free(hr_dev, srq); hns_roce_mtt_cleanup(hr_dev, &srq->mtt); - if (ibsrq->uobject) { + if (udata) { hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); - ib_umem_release(srq->idx_que.umem); - ib_umem_release(srq->umem); } else { kvfree(srq->wrid); hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift, &srq->buf); } + ib_umem_release(srq->idx_que.umem); + ib_umem_release(srq->umem); } int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 3100b0c31b0a..3c0c6aabc64e 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -2004,8 +2004,7 @@ static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct cqp_commands_info *cqp_info; u32 stag_idx; - if (iwmr->region) - ib_umem_release(iwmr->region); + ib_umem_release(iwmr->region); if (iwmr->type != IW_MEMREG_TYPE_MEM) { /* region is released. only test for userness. */ diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 72f238ddafb5..a7d238d312f0 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -277,9 +277,8 @@ err_dbmap: err_mtt: mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - if (udata) - ib_umem_release(cq->umem); - else + ib_umem_release(cq->umem); + if (!udata) mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); err_db: @@ -468,11 +467,8 @@ err_buf: kfree(cq->resize_buf); cq->resize_buf = NULL; - if (cq->resize_umem) { - ib_umem_release(cq->resize_umem); - cq->resize_umem = NULL; - } - + ib_umem_release(cq->resize_umem); + cq->resize_umem = NULL; out: mutex_unlock(&cq->resize_mutex); @@ -494,11 +490,11 @@ void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &mcq->db); - ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); } + ib_umem_release(mcq->umem); } static void dump_cqe(void *cqe) diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 520364defa28..82aff2f2fdc2 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1207,10 +1207,9 @@ err_mtt: mlx4_mtt_cleanup(dev->dev, &qp->mtt); err_buf: - if (qp->umem) - ib_umem_release(qp->umem); - else + if (!qp->umem) mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + ib_umem_release(qp->umem); err_db: if (!udata && qp_has_rq(init_attr)) @@ -1421,7 +1420,6 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, mlx4_ib_db_unmap_user(mcontext, &qp->db); } - ib_umem_release(qp->umem); } else { kvfree(qp->sq.wrid); kvfree(qp->rq.wrid); @@ -1432,6 +1430,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, if (qp->rq.wqe_cnt) mlx4_db_free(dev->dev, &qp->db); } + ib_umem_release(qp->umem); del_gid_entries(qp); } diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index c9f555e04c9f..848db7264cc9 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -204,10 +204,9 @@ err_mtt: mlx4_mtt_cleanup(dev->dev, &srq->mtt); err_buf: - if (srq->umem) - ib_umem_release(srq->umem); - else + if (!srq->umem) mlx4_buf_free(dev->dev, buf_size, &srq->buf); + ib_umem_release(srq->umem); err_db: if (!udata) @@ -275,13 +274,13 @@ void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &msrq->db); - ib_umem_release(msrq->umem); } else { kvfree(msrq->wrid); mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, &msrq->buf); mlx4_db_free(dev->dev, &msrq->db); } + ib_umem_release(msrq->umem); } void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 07b73df0e1a3..22230fd7d741 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1125,11 +1125,6 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, return 0; } -static void un_resize_user(struct mlx5_ib_cq *cq) -{ - ib_umem_release(cq->resize_umem); -} - static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, int entries, int cqe_size) { @@ -1152,12 +1147,6 @@ ex: return err; } -static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) -{ - free_cq_buf(dev, cq->resize_buf); - cq->resize_buf = NULL; -} - static int copy_resize_cqes(struct mlx5_ib_cq *cq) { struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); @@ -1338,10 +1327,11 @@ ex_alloc: kvfree(in); ex_resize: - if (udata) - un_resize_user(cq); - else - un_resize_kernel(dev, cq); + ib_umem_release(cq->resize_umem); + if (!udata) { + free_cq_buf(dev, cq->resize_buf); + cq->resize_buf = NULL; + } ex: mutex_unlock(&cq->resize_mutex); return err; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 4d033796dcfc..994abcebb057 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1507,10 +1507,9 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, return 0; err: - if (mr->umem) { - ib_umem_release(mr->umem); - mr->umem = NULL; - } + ib_umem_release(mr->umem); + mr->umem = NULL; + clean_mr(dev, mr); return err; } @@ -1630,10 +1629,10 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) * remove the DMA mapping. */ mlx5_mr_cache_free(dev, mr); - if (umem) { - ib_umem_release(umem); + ib_umem_release(umem); + if (umem) atomic_sub(npages, &dev->mdev->priv.reg_pages); - } + if (!mr->allocated_from_cache) kfree(mr); } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index ae847709b3d3..12ccbd584d2a 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -790,8 +790,7 @@ static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, atomic_dec(&dev->delay_drop.rqs_cnt); mlx5_ib_db_unmap_user(context, &rwq->db); - if (rwq->umem) - ib_umem_release(rwq->umem); + ib_umem_release(rwq->umem); } static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, @@ -977,8 +976,7 @@ err_free: kvfree(*in); err_umem: - if (ubuffer->umem) - ib_umem_release(ubuffer->umem); + ib_umem_release(ubuffer->umem); err_bfreg: if (bfregn != MLX5_IB_INVALID_BFREG) @@ -997,8 +995,7 @@ static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd, ibucontext); mlx5_ib_db_unmap_user(context, &qp->db); - if (base->ubuffer.umem) - ib_umem_release(base->ubuffer.umem); + ib_umem_release(base->ubuffer.umem); /* * Free only the BFREGs which are handled by the kernel. diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index d97124bee703..23554d8bf241 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -953,8 +953,7 @@ static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) struct mthca_mr *mmr = to_mmr(mr); mthca_free_mr(to_mdev(mr->device), mmr); - if (mmr->umem) - ib_umem_release(mmr->umem); + ib_umem_release(mmr->umem); kfree(mmr); return 0; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 10b35edb286b..bccc11378109 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -925,8 +925,7 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); /* it could be user registered memory. */ - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); /* Don't stop cleanup, in case FW is unresponsive */ diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 3fc7a4e901c3..27d90a84ea01 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1572,12 +1572,10 @@ qedr_iwarp_populate_user_qp(struct qedr_dev *dev, static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) { - if (qp->usq.umem) - ib_umem_release(qp->usq.umem); + ib_umem_release(qp->usq.umem); qp->usq.umem = NULL; - if (qp->urq.umem) - ib_umem_release(qp->urq.umem); + ib_umem_release(qp->urq.umem); qp->urq.umem = NULL; } @@ -2680,8 +2678,7 @@ int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); /* it could be user registered memory. */ - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index 38573fc0a9bf..7800e6930502 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -213,8 +213,7 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, err_page_dir: pvrdma_page_dir_cleanup(dev, &cq->pdir); err_umem: - if (!cq->is_kernel) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); err_cq: atomic_dec(&dev->num_cqs); return ret; @@ -226,8 +225,7 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) complete(&cq->free); wait_for_completion(&cq->free); - if (!cq->is_kernel) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); pvrdma_page_dir_cleanup(dev, &cq->pdir); } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 65dc47ffb8f3..f3a3d22ee8d7 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -290,8 +290,7 @@ int pvrdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) "could not deregister mem region, error: %d\n", ret); pvrdma_page_dir_cleanup(dev, &mr->pdir); - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr->pages); kfree(mr); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 0eaaead5baec..bca6a58a442e 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -391,12 +391,8 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, err_pdir: pvrdma_page_dir_cleanup(dev, &qp->pdir); err_umem: - if (!qp->is_kernel) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); err_qp: kfree(qp); atomic_dec(&dev->num_qps); @@ -429,12 +425,8 @@ static void pvrdma_free_qp(struct pvrdma_qp *qp) complete(&qp->free); wait_for_completion(&qp->free); - if (!qp->is_kernel) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); pvrdma_page_dir_cleanup(dev, &qp->pdir); -- cgit v1.2.3 From da3929218a4481fc12f9eaa30c9edb09aad5ff24 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 16 Jun 2019 15:05:58 +0300 Subject: RDMa/hns: Don't stuck in endless timeout loop The "end" variable is declared as unsigned and can't be negative, it leads to the situation where timeout limit is not honored, so let's convert logic to ensure that loop is bounded. drivers/infiniband/hw/hns/hns_roce_hw_v1.c: In function _hns_roce_v1_clear_hem_: drivers/infiniband/hw/hns/hns_roce_hw_v1.c:2471:12: warning: comparison of unsigned expression < 0 is always false [-Wtype-limits] 2471 | if (end < 0) { | ^ Fixes: 669cefb654cb ("RDMA/hns: Remove jiffies operation in disable interrupt context") Signed-off-by: Leon Romanovsky Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hns/hns_roce_hem.h | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index e865fc8a18a7..f1ccb8f35fe5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -34,8 +34,8 @@ #ifndef _HNS_ROCE_HEM_H #define _HNS_ROCE_HEM_H -#define HW_SYNC_TIMEOUT_MSECS 500 #define HW_SYNC_SLEEP_TIME_INTERVAL 20 +#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL) #define BT_CMD_SYNC_SHIFT 31 enum { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index cb004190ccba..2c0bc2536fda 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -2467,7 +2467,7 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, end = HW_SYNC_TIMEOUT_MSECS; while (1) { if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (end < 0) { + if (!end) { dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); -- cgit v1.2.3 From 4a9ceb7dbadf9e1435644b1f49720ee87431ce26 Mon Sep 17 00:00:00 2001 From: Mike Marciniszyn Date: Thu, 13 Jun 2019 08:30:52 -0400 Subject: IB/{rdmavt, qib, hfi1}: Convert to new completion API Convert all completions to use the new completion routine that fixes a race between post send and completion where fields from a SWQE can be read after SWQE has been freed. This patch also addresses issues reported in https://marc.info/?l=linux-kernel&m=155656897409107&w=2. The reserved operation path has no need for any barrier. The barrier for the other path is addressed by the smp_load_acquire() barrier. Cc: Andrea Parri Reviewed-by: Michael J. Ruhl Signed-off-by: Mike Marciniszyn Signed-off-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/infiniband/hw/hfi1/rc.c | 26 ++++---------------------- drivers/infiniband/hw/qib/qib_rc.c | 26 ++++---------------------- 2 files changed, 8 insertions(+), 44 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index a922edcf23d6..84b51cc36dbd 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -1819,23 +1819,14 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) } while (qp->s_last != qp->s_acked) { - u32 s_last; - wqe = rvt_get_swqe_ptr(qp, qp->s_last); if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) break; trdma_clean_swqe(qp, wqe); rvt_qp_wqe_unreserve(qp, wqe); - s_last = qp->s_last; - trace_hfi1_qp_send_completion(qp, wqe, s_last); - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_put_qp_swqe(qp, wqe); - rvt_qp_swqe_complete(qp, + trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); + rvt_qp_complete_swqe(qp, wqe, ib_hfi1_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); @@ -1879,19 +1870,10 @@ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, trace_hfi1_rc_completion(qp, wqe->lpsn); if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { - u32 s_last; - trdma_clean_swqe(qp, wqe); - rvt_put_qp_swqe(qp, wqe); rvt_qp_wqe_unreserve(qp, wqe); - s_last = qp->s_last; - trace_hfi1_qp_send_completion(qp, wqe, s_last); - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_qp_swqe_complete(qp, + trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); + rvt_qp_complete_swqe(qp, wqe, ib_hfi1_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index 2ac4c67f5ba1..8d9a94d6f685 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -921,20 +921,11 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) rvt_add_retry_timer(qp); while (qp->s_last != qp->s_acked) { - u32 s_last; - wqe = rvt_get_swqe_ptr(qp, qp->s_last); if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) break; - s_last = qp->s_last; - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_put_qp_swqe(qp, wqe); - rvt_qp_swqe_complete(qp, + rvt_qp_complete_swqe(qp, wqe, ib_qib_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); @@ -972,21 +963,12 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, * is finished. */ if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || - qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { - u32 s_last; - - rvt_put_qp_swqe(qp, wqe); - s_last = qp->s_last; - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_qp_swqe_complete(qp, + qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) + rvt_qp_complete_swqe(qp, wqe, ib_qib_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); - } else + else this_cpu_inc(*ibp->rvp.rc_delayed_comp); qp->s_retry = qp->s_retry_cnt; -- cgit v1.2.3 From 40ddb3f020834f9afb7aab31385994811f4db259 Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Thu, 13 Jun 2019 12:10:12 +0300 Subject: RDMA/efa: Use API to get contiguous memory blocks aligned to device supported page size Use the ib_umem_find_best_pgsz() and rdma_for_each_block() API when registering an MR instead of coding it in the driver. ib_umem_find_best_pgsz() is used to find the best suitable page size which replaces the existing efa_cont_pages() implementation. rdma_for_each_block() is used to iterate the umem in aligned contiguous memory blocks. Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Reviewed-by: Shiraz Saleem Signed-off-by: Gal Pressman Signed-off-by: Doug Ledford --- drivers/infiniband/hw/efa/efa_verbs.c | 88 +++++++++-------------------------- 1 file changed, 21 insertions(+), 67 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 5e6e5eb65cff..f187f1acdb55 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1011,21 +1011,15 @@ static int umem_to_page_list(struct efa_dev *dev, u8 hp_shift) { u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT); - struct sg_dma_page_iter sg_iter; - unsigned int page_idx = 0; + struct ib_block_iter biter; unsigned int hp_idx = 0; ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n", hp_cnt, pages_in_hp); - for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { - if (page_idx % pages_in_hp == 0) { - page_list[hp_idx] = sg_page_iter_dma_address(&sg_iter); - hp_idx++; - } - - page_idx++; - } + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + BIT(hp_shift)) + page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); return 0; } @@ -1356,56 +1350,6 @@ static int efa_create_pbl(struct efa_dev *dev, return 0; } -static void efa_cont_pages(struct ib_umem *umem, u64 addr, - unsigned long max_page_shift, - int *count, u8 *shift, u32 *ncont) -{ - struct scatterlist *sg; - u64 base = ~0, p = 0; - unsigned long tmp; - unsigned long m; - u64 len, pfn; - int i = 0; - int entry; - - addr = addr >> PAGE_SHIFT; - tmp = (unsigned long)addr; - m = find_first_bit(&tmp, BITS_PER_LONG); - if (max_page_shift) - m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m); - - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE); - pfn = sg_dma_address(sg) >> PAGE_SHIFT; - if (base + p != pfn) { - /* - * If either the offset or the new - * base are unaligned update m - */ - tmp = (unsigned long)(pfn | p); - if (!IS_ALIGNED(tmp, 1 << m)) - m = find_first_bit(&tmp, BITS_PER_LONG); - - base = pfn; - p = 0; - } - - p += len; - i += len; - } - - if (i) { - m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); - *ncont = DIV_ROUND_UP(i, (1 << m)); - } else { - m = 0; - *ncont = 0; - } - - *shift = PAGE_SHIFT + m; - *count = i; -} - struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata) @@ -1413,11 +1357,10 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, struct efa_dev *dev = to_edev(ibpd->device); struct efa_com_reg_mr_params params = {}; struct efa_com_reg_mr_result result = {}; - unsigned long max_page_shift; struct pbl_context pbl; + unsigned int pg_sz; struct efa_mr *mr; int inline_size; - int npages; int err; if (udata->inlen && @@ -1454,13 +1397,24 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, params.iova = virt_addr; params.mr_length_in_bytes = length; params.permissions = access_flags & 0x1; - max_page_shift = fls64(dev->dev_attr.page_size_cap); - efa_cont_pages(mr->umem, start, max_page_shift, &npages, - ¶ms.page_shift, ¶ms.page_num); + pg_sz = ib_umem_find_best_pgsz(mr->umem, + dev->dev_attr.page_size_cap, + virt_addr); + if (!pg_sz) { + err = -EOPNOTSUPP; + ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n", + dev->dev_attr.page_size_cap); + goto err_unmap; + } + + params.page_shift = __ffs(pg_sz); + params.page_num = DIV_ROUND_UP(length + (start & (pg_sz - 1)), + pg_sz); + ibdev_dbg(&dev->ibdev, - "start %#llx length %#llx npages %d params.page_shift %u params.page_num %u\n", - start, length, npages, params.page_shift, params.page_num); + "start %#llx length %#llx params.page_shift %u params.page_num %u\n", + start, length, params.page_shift, params.page_num); inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array); if (params.page_num <= inline_size) { -- cgit v1.2.3 From b41f75724ab8aeeba1001e63c31db8623432f001 Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Thu, 13 Jun 2019 12:10:13 +0300 Subject: RDMA/efa: Be consistent with success flow return value The EFA driver is written with success oriented flows in mind, meaning that functions should mostly end with a return 0 statement. Error flows return their error value on their own instead of assuming that the function will return the error at the end. This commit fixes a bunch of functions that were not aligned with this behavior. Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Doug Ledford --- drivers/infiniband/hw/efa/efa_com_cmd.c | 4 ++-- drivers/infiniband/hw/efa/efa_main.c | 2 +- drivers/infiniband/hw/efa/efa_verbs.c | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 91e7f2195802..d2464c8390bb 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -56,7 +56,7 @@ int efa_com_create_qp(struct efa_com_dev *edev, res->send_sub_cq_idx = cmd_completion.send_sub_cq_idx; res->recv_sub_cq_idx = cmd_completion.recv_sub_cq_idx; - return err; + return 0; } int efa_com_modify_qp(struct efa_com_dev *edev, @@ -178,7 +178,7 @@ int efa_com_create_cq(struct efa_com_dev *edev, result->cq_idx = cmd_completion.cq_idx; result->actual_depth = params->cq_depth; - return err; + return 0; } int efa_com_destroy_cq(struct efa_com_dev *edev, diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 46861461dd2d..dd1c6d49466f 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -100,7 +100,7 @@ static int efa_request_mgmnt_irq(struct efa_dev *dev) nr_cpumask_bits, &irq->affinity_hint_mask, irq->vector); irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask); - return err; + return 0; } static void efa_setup_mgmnt_irq(struct efa_dev *dev) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index f187f1acdb55..7b4e0fa99817 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1594,13 +1594,15 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL; } - if (err) + if (err) { ibdev_dbg( &dev->ibdev, "Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n", entry->address, length, entry->mmap_flag, err); + return err; + } - return err; + return 0; } int efa_mmap(struct ib_ucontext *ibucontext, -- cgit v1.2.3 From 4b06843d404855063decbccd206d3dc927280b0c Mon Sep 17 00:00:00 2001 From: Firas Jahjah Date: Thu, 13 Jun 2019 12:10:14 +0300 Subject: RDMA/efa: Print address on AH creation failure For debugging purposes, print destination address if failed to create AH. Signed-off-by: Firas Jahjah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Doug Ledford --- drivers/infiniband/hw/efa/efa_com_cmd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index d2464c8390bb..fad5c2ee7bb1 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -300,7 +300,8 @@ int efa_com_create_ah(struct efa_com_dev *edev, (struct efa_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (err) { - ibdev_err(edev->efa_dev, "Failed to create ah [%d]\n", err); + ibdev_err(edev->efa_dev, "Failed to create ah for %pI6 [%d]\n", + ah_cmd.dest_addr, err); return err; } -- cgit v1.2.3 From 6c984472bad12da18b88e9f4345f4970bbec0b3e Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:42 +0300 Subject: RDMA/mlx5: Implement mlx5_ib_map_mr_sg_pi and mlx5_ib_alloc_mr_integrity mlx5_ib_map_mr_sg_pi() will map the PI and data dma mapped SG lists to the mlx5 memory region prior to the registration operation. In the new API, the mlx5 driver will allocate an internal memory region for the UMR operation to register both PI and data SG lists. The internal MR will use KLM mode in order to map 2 (possibly non-contiguous/non-align) SG lists using 1 memory key. In the new API, each ULP will use 1 memory region for the signature operation (instead of 3 in the old API). This memory region will have a key that will be exposed to remote server to perform RDMA operation. The internal memory key that will map the SG lists will stay private. Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 11 +++ drivers/infiniband/hw/mlx5/mr.c | 187 ++++++++++++++++++++++++++++++++--- 3 files changed, 189 insertions(+), 11 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 3b1985215cb9..23fedff9f080 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6146,6 +6146,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .add_gid = mlx5_ib_add_gid, .alloc_mr = mlx5_ib_alloc_mr, + .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity, .alloc_pd = mlx5_ib_alloc_pd, .alloc_ucontext = mlx5_ib_alloc_ucontext, .attach_mcast = mlx5_ib_mcg_attach, @@ -6175,6 +6176,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .get_dma_mr = mlx5_ib_get_dma_mr, .get_link_layer = mlx5_ib_port_link_layer, .map_mr_sg = mlx5_ib_map_mr_sg, + .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi, .mmap = mlx5_ib_mmap, .modify_cq = mlx5_ib_modify_cq, .modify_device = mlx5_ib_modify_device, diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 82cfe86087b6..7980814f355d 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -587,6 +587,9 @@ struct mlx5_ib_mr { void *descs; dma_addr_t desc_map; int ndescs; + int data_length; + int meta_ndescs; + int meta_length; int max_descs; int desc_size; int access_mode; @@ -605,6 +608,7 @@ struct mlx5_ib_mr { int access_flags; /* Needed for rereg MR */ struct mlx5_ib_mr *parent; + struct mlx5_ib_mr *pi_mr; /* Needed for IB_MR_TYPE_INTEGRITY */ atomic_t num_leaf_free; wait_queue_head_t q_leaf_free; struct mlx5_async_work cb_work; @@ -1148,8 +1152,15 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata); +struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, + u32 max_num_sg, + u32 max_num_meta_sg); int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); +int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset); int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad_hdr *in, size_t in_mad_size, diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 994abcebb057..af8ae1e76fd4 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1639,16 +1639,22 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { - dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr)); + struct mlx5_ib_mr *mmr = to_mmr(ibmr); + + if (ibmr->type == IB_MR_TYPE_INTEGRITY) + dereg_mr(to_mdev(mmr->pi_mr->ibmr.device), mmr->pi_mr); + + dereg_mr(to_mdev(ibmr->device), mmr); + return 0; } -struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata) +static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, + u32 max_num_sg, u32 max_num_meta_sg) { struct mlx5_ib_dev *dev = to_mdev(pd->device); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - int ndescs = ALIGN(max_num_sg, 4); + int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); struct mlx5_ib_mr *mr; void *mkc; u32 *in; @@ -1670,8 +1676,72 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; + + err = mlx5_alloc_priv_descs(pd->device, mr, + ndescs, sizeof(struct mlx5_klm)); + if (err) + goto err_free_in; + mr->desc_size = sizeof(struct mlx5_klm); + mr->max_descs = ndescs; + + MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); + MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); + MLX5_SET(mkc, mkc, umr_en, 1); + + mr->ibmr.pd = pd; + mr->ibmr.device = pd->device; + err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); + if (err) + goto err_priv_descs; + + mr->mmkey.type = MLX5_MKEY_MR; + mr->ibmr.lkey = mr->mmkey.key; + mr->ibmr.rkey = mr->mmkey.key; + mr->umem = NULL; + kfree(in); + + return mr; + +err_priv_descs: + mlx5_free_priv_descs(mr); +err_free_in: + kfree(in); +err_free: + kfree(mr); + return ERR_PTR(err); +} + +static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, + enum ib_mr_type mr_type, u32 max_num_sg, + u32 max_num_meta_sg) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + int ndescs = ALIGN(max_num_sg, 4); + struct mlx5_ib_mr *mr; + void *mkc; + u32 *in; + int err; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); + + in = kzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_free; + } + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, free, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + if (mr_type == IB_MR_TYPE_MEM_REG) { mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; + MLX5_SET(mkc, mkc, translations_octword_size, ndescs); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, sizeof(struct mlx5_mtt)); @@ -1682,6 +1752,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, mr->max_descs = ndescs; } else if (mr_type == IB_MR_TYPE_SG_GAPS) { mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; + MLX5_SET(mkc, mkc, translations_octword_size, ndescs); err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, sizeof(struct mlx5_klm)); @@ -1689,11 +1760,13 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, goto err_free_in; mr->desc_size = sizeof(struct mlx5_klm); mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_SIGNATURE) { + } else if (mr_type == IB_MR_TYPE_SIGNATURE || + mr_type == IB_MR_TYPE_INTEGRITY) { u32 psv_index[2]; MLX5_SET(mkc, mkc, bsf_en, 1); MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); + MLX5_SET(mkc, mkc, translations_octword_size, 4); mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); if (!mr->sig) { err = -ENOMEM; @@ -1714,6 +1787,14 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, mr->sig->sig_err_exists = false; /* Next UMR, Arm SIGERR */ ++mr->sig->sigerr_count; + if (mr_type == IB_MR_TYPE_INTEGRITY) { + mr->pi_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, + max_num_meta_sg); + if (IS_ERR(mr->pi_mr)) { + err = PTR_ERR(mr->pi_mr); + goto err_destroy_psv; + } + } } else { mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); err = -EINVAL; @@ -1727,7 +1808,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, mr->ibmr.device = pd->device; err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); if (err) - goto err_destroy_psv; + goto err_free_pi_mr; mr->mmkey.type = MLX5_MKEY_MR; mr->ibmr.lkey = mr->mmkey.key; @@ -1737,6 +1818,11 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, return &mr->ibmr; +err_free_pi_mr: + if (mr->pi_mr) { + dereg_mr(to_mdev(mr->pi_mr->ibmr.device), mr->pi_mr); + mr->pi_mr = NULL; + } err_destroy_psv: if (mr->sig) { if (mlx5_core_destroy_psv(dev->mdev, @@ -1758,6 +1844,19 @@ err_free: return ERR_PTR(err); } +struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, + u32 max_num_sg, struct ib_udata *udata) +{ + return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); +} + +struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, + u32 max_num_sg, u32 max_num_meta_sg) +{ + return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, + max_num_meta_sg); +} + struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata) { @@ -1890,13 +1989,16 @@ static int mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, struct scatterlist *sgl, unsigned short sg_nents, - unsigned int *sg_offset_p) + unsigned int *sg_offset_p, + struct scatterlist *meta_sgl, + unsigned short meta_sg_nents, + unsigned int *meta_sg_offset_p) { struct scatterlist *sg = sgl; struct mlx5_klm *klms = mr->descs; unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; u32 lkey = mr->ibmr.pd->local_dma_lkey; - int i; + int i, j = 0; mr->ibmr.iova = sg_dma_address(sg) + sg_offset; mr->ibmr.length = 0; @@ -1911,12 +2013,36 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, sg_offset = 0; } - mr->ndescs = i; if (sg_offset_p) *sg_offset_p = sg_offset; - return i; + mr->ndescs = i; + mr->data_length = mr->ibmr.length; + + if (meta_sg_nents) { + sg = meta_sgl; + sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0; + for_each_sg(meta_sgl, sg, meta_sg_nents, j) { + if (unlikely(i + j >= mr->max_descs)) + break; + klms[i + j].va = cpu_to_be64(sg_dma_address(sg) + + sg_offset); + klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - + sg_offset); + klms[i + j].key = cpu_to_be32(lkey); + mr->ibmr.length += sg_dma_len(sg) - sg_offset; + + sg_offset = 0; + } + if (meta_sg_offset_p) + *meta_sg_offset_p = sg_offset; + + mr->meta_ndescs = j; + mr->meta_length = mr->ibmr.length - mr->data_length; + } + + return i + j; } static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) @@ -1933,6 +2059,44 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) return 0; } +int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = mr->pi_mr; + int n; + + WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); + + pi_mr->ndescs = 0; + pi_mr->meta_ndescs = 0; + pi_mr->meta_length = 0; + + ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, + meta_sg, meta_sg_nents, meta_sg_offset); + + /* This is zero-based memory region */ + pi_mr->ibmr.iova = 0; + ibmr->length = pi_mr->ibmr.length; + ibmr->iova = pi_mr->ibmr.iova; + ibmr->sig_attrs->meta_length = pi_mr->meta_length; + + ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + if (unlikely(n != data_sg_nents + meta_sg_nents)) + return -ENOMEM; + + return 0; +} + int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { @@ -1946,7 +2110,8 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, DMA_TO_DEVICE); if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) - n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset); + n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0, + NULL); else n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, mlx5_set_page); -- cgit v1.2.3 From 62e3c379d4d713dbcb70adc7c65443fd8722b33f Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:43 +0300 Subject: RDMA/mlx5: Add attr for max number page list length for PI operation PI offload (protection information) is a feature that each RDMA provider can implement differently. Thus, introduce new device attribute to define the maximal length of the page list for PI fast registration operation. For example, mlx5 driver uses a single internal MR to map both data and protection SGL's, so it's equal to max_fast_reg_page_list_len / 2. Signed-off-by: Max Gurtovoy Reviewed-by: Leon Romanovsky Reviewed-by: Christoph Hellwig Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 23fedff9f080..bd0322b61362 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1009,6 +1009,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, props->max_srq_sge = max_rq_sg - 1; props->max_fast_reg_page_list_len = 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size); + props->max_pi_fast_reg_page_list_len = + props->max_fast_reg_page_list_len / 2; get_atomic_caps_qp(dev, props); props->masked_atomic_cap = IB_ATOMIC_NONE; props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg); -- cgit v1.2.3 From 9ac7c4bcd383adbac9ad02ec033cce9a3a261d28 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:44 +0300 Subject: RDMA/mlx5: Pass UMR segment flags instead of boolean UMR ctrl segment flags can vary between UMR operations. for example, using inline UMR or adding free/not-free checks for a memory key. This is a preparation commit before adding new signature API that will not need not-free checks for the internal memory key during the UMR operation. Signed-off-by: Max Gurtovoy Reviewed-by: Leon Romanovsky Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 12ccbd584d2a..6cb94bf507b4 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4167,15 +4167,13 @@ static __be64 sig_mkey_mask(void) } static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, - struct mlx5_ib_mr *mr, bool umr_inline) + struct mlx5_ib_mr *mr, u8 flags) { int size = mr->ndescs * mr->desc_size; memset(umr, 0, sizeof(*umr)); - umr->flags = MLX5_UMR_CHECK_NOT_FREE; - if (umr_inline) - umr->flags |= MLX5_UMR_INLINE; + umr->flags = flags; umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size)); umr->mkey_mask = frwr_mkey_mask(); } @@ -4756,12 +4754,14 @@ static int set_psv_wr(struct ib_sig_domain *domain, static int set_reg_wr(struct mlx5_ib_qp *qp, const struct ib_reg_wr *wr, - void **seg, int *size, void **cur_edge) + void **seg, int *size, void **cur_edge, + bool check_not_free) { struct mlx5_ib_mr *mr = to_mmr(wr->mr); struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); size_t mr_list_size = mr->ndescs * mr->desc_size; bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD; + u8 flags = 0; if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { mlx5_ib_warn(to_mdev(qp->ibqp.device), @@ -4769,7 +4769,12 @@ static int set_reg_wr(struct mlx5_ib_qp *qp, return -EINVAL; } - set_reg_umr_seg(*seg, mr, umr_inline); + if (check_not_free) + flags |= MLX5_UMR_CHECK_NOT_FREE; + if (umr_inline) + flags |= MLX5_UMR_INLINE; + + set_reg_umr_seg(*seg, mr, flags); *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); @@ -5000,7 +5005,7 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, qp->sq.wr_data[idx] = IB_WR_REG_MR; ctrl->imm = cpu_to_be32(reg_wr(wr)->key); err = set_reg_wr(qp, reg_wr(wr), &seg, &size, - &cur_edge); + &cur_edge, true); if (err) { *bad_wr = wr; goto out; -- cgit v1.2.3 From 22465bba39c001ddac735531074d3d4d926d9088 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:45 +0300 Subject: RDMA/mlx5: Update set_sig_data_segment attribute for new signature API Explicitly pass the sig_mr and the access flags for the mkey segment configuration. This function will be used also in the new signature API, so modify it in order to use it in both APIs. This is a preparation commit before adding new signature API. Signed-off-by: Max Gurtovoy Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 6cb94bf507b4..ce8fccb04c3c 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4645,17 +4645,15 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, } static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, - const struct ib_sig_handover_wr *wr, u32 size, - u32 length, u32 pdn) + struct ib_mr *sig_mr, int access_flags, + u32 size, u32 length, u32 pdn) { - struct ib_mr *sig_mr = wr->sig_mr; u32 sig_key = sig_mr->rkey; u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1; memset(seg, 0, sizeof(*seg)); - seg->flags = get_umr_flags(wr->access_flags) | - MLX5_MKC_ACCESS_MODE_KLMS; + seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS; seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00); seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 | MLX5_MKEY_BSF_EN | pdn); @@ -4712,7 +4710,8 @@ static int set_sig_umr_wr(const struct ib_send_wr *send_wr, *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn); + set_sig_mkey_segment(*seg, wr->sig_mr, wr->access_flags, xlt_size, + region_len, pdn); *seg += sizeof(struct mlx5_mkey_seg); *size += sizeof(struct mlx5_mkey_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); -- cgit v1.2.3 From 38ca87c6f1e514686d4a385246d1afe1e1f2e482 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:46 +0300 Subject: RDMA/mlx5: Introduce and implement new IB_WR_REG_MR_INTEGRITY work request This new WR will be used to perform PI (protection information) handover using the new API. Using the new API, the user will post a single WR that will internally perform all the needed actions to complete PI operation. This new WR will use a memory region that was allocated as IB_MR_TYPE_INTEGRITY and was mapped using ib_map_mr_sg_pi to perform the registration. In the old API, in order to perform a signature handover operation, each ULP should perform the following: 1. Map and register the data buffers. 2. Map and register the protection buffers. 3. Post a special reg WR to configure the signature handover operation layout. 4. Invalidate the signature memory key. 5. Invalidate protection buffers memory key. 6. Invalidate data buffers memory key. In the new API, the mapping of both data and protection buffers is performed using a single call to ib_map_mr_sg_pi function. Also the registration of the buffers and the configuration of the signature operation layout is done by a single new work request called IB_WR_REG_MR_INTEGRITY. This patch implements this operation for mlx5 devices that are capable to offload data integrity generation/validation while performing the actual buffer transfer. This patch will not remove the old signature API that is used by the iSER initiator and target drivers. This will be done in the future. In the internal implementation, for each IB_WR_REG_MR_INTEGRITY work request, we are using a single UMR operation to register both data and protection buffers using KLM's. Afterwards, another UMR operation will describe the strided block format. These will be followed by 2 SET_PSV operations to set the memory/wire domains initial signature parameters passed by the user. In the end of the whole transaction, only the signature memory key (the one that exposed for the RDMA operation) will be invalidated. Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/qp.c | 218 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 198 insertions(+), 20 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index ce8fccb04c3c..f6651b93e469 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4169,7 +4169,7 @@ static __be64 sig_mkey_mask(void) static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, struct mlx5_ib_mr *mr, u8 flags) { - int size = mr->ndescs * mr->desc_size; + int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; memset(umr, 0, sizeof(*umr)); @@ -4300,7 +4300,7 @@ static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg, struct mlx5_ib_mr *mr, u32 key, int access) { - int ndescs = ALIGN(mr->ndescs, 8) >> 1; + int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1; memset(seg, 0, sizeof(*seg)); @@ -4351,7 +4351,7 @@ static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg, struct mlx5_ib_mr *mr, struct mlx5_ib_pd *pd) { - int bcount = mr->desc_size * mr->ndescs; + int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs); dseg->addr = cpu_to_be64(mr->desc_map); dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64)); @@ -4544,23 +4544,52 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr, return 0; } -static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, - struct mlx5_ib_qp *qp, void **seg, - int *size, void **cur_edge) +static int set_sig_data_segment(const struct ib_send_wr *send_wr, + struct ib_mr *sig_mr, + struct ib_sig_attrs *sig_attrs, + struct mlx5_ib_qp *qp, void **seg, int *size, + void **cur_edge) { - struct ib_sig_attrs *sig_attrs = wr->sig_attrs; - struct ib_mr *sig_mr = wr->sig_mr; struct mlx5_bsf *bsf; - u32 data_len = wr->wr.sg_list->length; - u32 data_key = wr->wr.sg_list->lkey; - u64 data_va = wr->wr.sg_list->addr; + u32 data_len; + u32 data_key; + u64 data_va; + u32 prot_len = 0; + u32 prot_key = 0; + u64 prot_va = 0; + bool prot = false; int ret; int wqe_size; - if (!wr->prot || - (data_key == wr->prot->lkey && - data_va == wr->prot->addr && - data_len == wr->prot->length)) { + if (send_wr->opcode == IB_WR_REG_SIG_MR) { + const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr); + + data_len = wr->wr.sg_list->length; + data_key = wr->wr.sg_list->lkey; + data_va = wr->wr.sg_list->addr; + if (wr->prot) { + prot_len = wr->prot->length; + prot_key = wr->prot->lkey; + prot_va = wr->prot->addr; + prot = true; + } + } else { + struct mlx5_ib_mr *mr = to_mmr(sig_mr); + struct mlx5_ib_mr *pi_mr = mr->pi_mr; + + data_len = pi_mr->data_length; + data_key = pi_mr->ibmr.lkey; + data_va = pi_mr->ibmr.iova; + if (pi_mr->meta_ndescs) { + prot_len = pi_mr->meta_length; + prot_key = pi_mr->ibmr.lkey; + prot_va = pi_mr->ibmr.iova + data_len; + prot = true; + } + } + + if (!prot || (data_key == prot_key && data_va == prot_va && + data_len == prot_len)) { /** * Source domain doesn't contain signature information * or data and protection are interleaved in memory. @@ -4594,8 +4623,6 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, struct mlx5_stride_block_ctrl_seg *sblock_ctrl; struct mlx5_stride_block_entry *data_sentry; struct mlx5_stride_block_entry *prot_sentry; - u32 prot_key = wr->prot->lkey; - u64 prot_va = wr->prot->addr; u16 block_size = sig_attrs->mem.sig.dif.pi_interval; int prot_size; @@ -4673,6 +4700,56 @@ static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, umr->mkey_mask = sig_mkey_mask(); } +static int set_pi_umr_wr(const struct ib_send_wr *send_wr, + struct mlx5_ib_qp *qp, void **seg, int *size, + void **cur_edge) +{ + const struct ib_reg_wr *wr = reg_wr(send_wr); + struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr); + struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr; + struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs; + u32 pdn = get_pd(qp)->pdn; + u32 xlt_size; + int region_len, ret; + + if (unlikely(send_wr->num_sge != 0) || + unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || + unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || + unlikely(!sig_mr->sig->sig_status_checked)) + return -EINVAL; + + /* length of the protected region, data + protection */ + region_len = pi_mr->ibmr.length; + + /** + * KLM octoword size - if protection was provided + * then we use strided block format (3 octowords), + * else we use single KLM (1 octoword) + **/ + if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE) + xlt_size = 0x30; + else + xlt_size = sizeof(struct mlx5_klm); + + set_sig_umr_segment(*seg, xlt_size); + *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); + *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; + handle_post_send_edge(&qp->sq, seg, *size, cur_edge); + + set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len, + pdn); + *seg += sizeof(struct mlx5_mkey_seg); + *size += sizeof(struct mlx5_mkey_seg) / 16; + handle_post_send_edge(&qp->sq, seg, *size, cur_edge); + + ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, + cur_edge); + if (ret) + return ret; + + sig_mr->sig->sig_status_checked = false; + return 0; +} static int set_sig_umr_wr(const struct ib_send_wr *send_wr, struct mlx5_ib_qp *qp, void **seg, int *size, @@ -4716,7 +4793,8 @@ static int set_sig_umr_wr(const struct ib_send_wr *send_wr, *size += sizeof(struct mlx5_mkey_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - ret = set_sig_data_segment(wr, qp, seg, size, cur_edge); + ret = set_sig_data_segment(send_wr, wr->sig_mr, wr->sig_attrs, qp, seg, + size, cur_edge); if (ret) return ret; @@ -4758,7 +4836,7 @@ static int set_reg_wr(struct mlx5_ib_qp *qp, { struct mlx5_ib_mr *mr = to_mmr(wr->mr); struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); - size_t mr_list_size = mr->ndescs * mr->desc_size; + int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD; u8 flags = 0; @@ -4899,8 +4977,11 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */ struct mlx5_ib_dev *dev = to_mdev(ibqp->device); struct mlx5_core_dev *mdev = dev->mdev; + struct ib_reg_wr reg_pi_wr; struct mlx5_ib_qp *qp; struct mlx5_ib_mr *mr; + struct mlx5_ib_mr *pi_mr; + struct ib_sig_attrs *sig_attrs; struct mlx5_wqe_xrc_seg *xrc; struct mlx5_bf *bf; void *cur_edge; @@ -4954,7 +5035,8 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, goto out; } - if (wr->opcode == IB_WR_REG_MR) { + if (wr->opcode == IB_WR_REG_MR || + wr->opcode == IB_WR_REG_MR_INTEGRITY) { fence = dev->umr_fence; next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; } else { @@ -5012,6 +5094,102 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, num_sge = 0; break; + case IB_WR_REG_MR_INTEGRITY: + memset(®_pi_wr, 0, sizeof(struct ib_reg_wr)); + + mr = to_mmr(reg_wr(wr)->mr); + pi_mr = mr->pi_mr; + + reg_pi_wr.mr = &pi_mr->ibmr; + reg_pi_wr.access = reg_wr(wr)->access; + reg_pi_wr.key = pi_mr->ibmr.rkey; + + qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY; + ctrl->imm = cpu_to_be32(reg_pi_wr.key); + /* UMR for data + protection registration */ + err = set_reg_wr(qp, ®_pi_wr, &seg, &size, + &cur_edge, false); + if (err) { + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, cur_edge, idx, + wr->wr_id, nreq, fence, + MLX5_OPCODE_UMR); + + err = begin_wqe(qp, &seg, &ctrl, wr, &idx, + &size, &cur_edge, nreq); + if (err) { + mlx5_ib_warn(dev, "\n"); + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + ctrl->imm = cpu_to_be32(mr->ibmr.rkey); + /* UMR for sig MR */ + err = set_pi_umr_wr(wr, qp, &seg, &size, + &cur_edge); + if (err) { + mlx5_ib_warn(dev, "\n"); + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, cur_edge, idx, + wr->wr_id, nreq, fence, + MLX5_OPCODE_UMR); + + /* + * SET_PSV WQEs are not signaled and solicited + * on error + */ + sig_attrs = mr->ibmr.sig_attrs; + err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, + &size, &cur_edge, nreq, false, + true); + if (err) { + mlx5_ib_warn(dev, "\n"); + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + err = set_psv_wr(&sig_attrs->mem, + mr->sig->psv_memory.psv_idx, + &seg, &size); + if (err) { + mlx5_ib_warn(dev, "\n"); + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, cur_edge, idx, + wr->wr_id, nreq, next_fence, + MLX5_OPCODE_SET_PSV); + + err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, + &size, &cur_edge, nreq, false, + true); + if (err) { + mlx5_ib_warn(dev, "\n"); + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + err = set_psv_wr(&sig_attrs->wire, + mr->sig->psv_wire.psv_idx, + &seg, &size); + if (err) { + mlx5_ib_warn(dev, "\n"); + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, cur_edge, idx, + wr->wr_id, nreq, next_fence, + MLX5_OPCODE_SET_PSV); + + qp->next_fence = + MLX5_FENCE_MODE_INITIATOR_SMALL; + num_sge = 0; + goto skip_psv; + case IB_WR_REG_SIG_MR: qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR; mr = to_mmr(sig_handover_wr(wr)->sig_mr); -- cgit v1.2.3 From c0a6cbb9cbccffc249743afa16e64f16c46c80b2 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 11 Jun 2019 18:52:50 +0300 Subject: RDMA/core: Rename signature qp create flag and signature device capability Rename IB_QP_CREATE_SIGNATURE_EN to IB_QP_CREATE_INTEGRITY_EN and IB_DEVICE_SIGNATURE_HANDOVER to IB_DEVICE_INTEGRITY_HANDOVER. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Reviewed-by: Christoph Hellwig Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +-- drivers/infiniband/hw/mlx5/qp.c | 14 +++++++------- 3 files changed, 9 insertions(+), 10 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index bd0322b61362..9aa10cfbc064 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -889,7 +889,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; if (MLX5_CAP_GEN(mdev, sho)) { - props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER; + props->device_cap_flags |= IB_DEVICE_INTEGRITY_HANDOVER; /* At this stage no support for signature handover */ props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 | IB_PROT_T10DIF_TYPE_2 | diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 7980814f355d..5999792b5698 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -431,8 +431,7 @@ struct mlx5_ib_qp { int create_type; - /* Store signature errors */ - bool signature_en; + bool integrity_en; struct list_head qps_list; struct list_head cq_recv_list; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index f6651b93e469..879162da63e3 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -442,9 +442,9 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr) } size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); - if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN && + if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN && ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE) - return MLX5_SIG_WQE_SIZE; + return MLX5_SIG_WQE_SIZE; else return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); } @@ -496,8 +496,8 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, sizeof(struct mlx5_wqe_inline_seg); attr->cap.max_inline_data = qp->max_inline_data; - if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) - qp->signature_en = true; + if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) + qp->integrity_en = true; wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; @@ -1039,7 +1039,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, void *qpc; int err; - if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | + if (init_attr->create_flags & ~(IB_QP_CREATE_INTEGRITY_EN | IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | IB_QP_CREATE_IPOIB_UD_LSO | IB_QP_CREATE_NETIF_QP | @@ -4714,7 +4714,7 @@ static int set_pi_umr_wr(const struct ib_send_wr *send_wr, if (unlikely(send_wr->num_sge != 0) || unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || + unlikely(!sig_mr->sig) || unlikely(!qp->integrity_en) || unlikely(!sig_mr->sig->sig_status_checked)) return -EINVAL; @@ -4763,7 +4763,7 @@ static int set_sig_umr_wr(const struct ib_send_wr *send_wr, if (unlikely(wr->wr.num_sge != 1) || unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || + unlikely(!sig_mr->sig) || unlikely(!qp->integrity_en) || unlikely(!sig_mr->sig->sig_status_checked)) return -EINVAL; -- cgit v1.2.3 From 185eddc45798b9f73e5470964948d79b4c8df4b7 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:51 +0300 Subject: RDMA/core: Validate integrity handover device cap Protect the case that a ULP tries to allocate a QP with signature enabled flag while the LLD doesn't support this feature. While we're here, also move integrity_en attribute from mlx5_qp to ib_qp as a preparation for adding new integrity API to the rw-API (that is part of ib_core module). Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: Christoph Hellwig Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 -- drivers/infiniband/hw/mlx5/qp.c | 7 ++----- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 5999792b5698..d418219e68c6 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -431,8 +431,6 @@ struct mlx5_ib_qp { int create_type; - bool integrity_en; - struct list_head qps_list; struct list_head cq_recv_list; struct list_head cq_send_list; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 879162da63e3..d77a64c551ea 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -496,9 +496,6 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, sizeof(struct mlx5_wqe_inline_seg); attr->cap.max_inline_data = qp->max_inline_data; - if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) - qp->integrity_en = true; - wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { @@ -4714,7 +4711,7 @@ static int set_pi_umr_wr(const struct ib_send_wr *send_wr, if (unlikely(send_wr->num_sge != 0) || unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->integrity_en) || + unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || unlikely(!sig_mr->sig->sig_status_checked)) return -EINVAL; @@ -4763,7 +4760,7 @@ static int set_sig_umr_wr(const struct ib_send_wr *send_wr, if (unlikely(wr->wr.num_sge != 1) || unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->integrity_en) || + unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || unlikely(!sig_mr->sig->sig_status_checked)) return -EINVAL; -- cgit v1.2.3 From 5c171cbe3ab3d1390290eaa85e7b371cc26b1122 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 11 Jun 2019 18:52:54 +0300 Subject: RDMA/mlx5: Remove unused IB_WR_REG_SIG_MR code IB_WR_REG_SIG_MR is not needed after IB_WR_REG_MR_INTEGRITY was used. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 15 ++- drivers/infiniband/hw/mlx5/qp.c | 154 ++---------------------------- drivers/infiniband/hw/vmw_pvrdma/pvrdma.h | 2 +- 3 files changed, 17 insertions(+), 154 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index af8ae1e76fd4..36d1d6f8bb47 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1760,8 +1760,7 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, goto err_free_in; mr->desc_size = sizeof(struct mlx5_klm); mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_SIGNATURE || - mr_type == IB_MR_TYPE_INTEGRITY) { + } else if (mr_type == IB_MR_TYPE_INTEGRITY) { u32 psv_index[2]; MLX5_SET(mkc, mkc, bsf_en, 1); @@ -1787,13 +1786,11 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, mr->sig->sig_err_exists = false; /* Next UMR, Arm SIGERR */ ++mr->sig->sigerr_count; - if (mr_type == IB_MR_TYPE_INTEGRITY) { - mr->pi_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, - max_num_meta_sg); - if (IS_ERR(mr->pi_mr)) { - err = PTR_ERR(mr->pi_mr); - goto err_destroy_psv; - } + mr->pi_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, + max_num_meta_sg); + if (IS_ERR(mr->pi_mr)) { + err = PTR_ERR(mr->pi_mr); + goto err_destroy_psv; } } else { mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index d77a64c551ea..60536c9c008f 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4557,32 +4557,17 @@ static int set_sig_data_segment(const struct ib_send_wr *send_wr, bool prot = false; int ret; int wqe_size; + struct mlx5_ib_mr *mr = to_mmr(sig_mr); + struct mlx5_ib_mr *pi_mr = mr->pi_mr; - if (send_wr->opcode == IB_WR_REG_SIG_MR) { - const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr); - - data_len = wr->wr.sg_list->length; - data_key = wr->wr.sg_list->lkey; - data_va = wr->wr.sg_list->addr; - if (wr->prot) { - prot_len = wr->prot->length; - prot_key = wr->prot->lkey; - prot_va = wr->prot->addr; - prot = true; - } - } else { - struct mlx5_ib_mr *mr = to_mmr(sig_mr); - struct mlx5_ib_mr *pi_mr = mr->pi_mr; - - data_len = pi_mr->data_length; - data_key = pi_mr->ibmr.lkey; - data_va = pi_mr->ibmr.iova; - if (pi_mr->meta_ndescs) { - prot_len = pi_mr->meta_length; - prot_key = pi_mr->ibmr.lkey; - prot_va = pi_mr->ibmr.iova + data_len; - prot = true; - } + data_len = pi_mr->data_length; + data_key = pi_mr->ibmr.lkey; + data_va = pi_mr->ibmr.iova; + if (pi_mr->meta_ndescs) { + prot_len = pi_mr->meta_length; + prot_key = pi_mr->ibmr.lkey; + prot_va = pi_mr->ibmr.iova + data_len; + prot = true; } if (!prot || (data_key == prot_key && data_va == prot_va && @@ -4748,57 +4733,6 @@ static int set_pi_umr_wr(const struct ib_send_wr *send_wr, return 0; } -static int set_sig_umr_wr(const struct ib_send_wr *send_wr, - struct mlx5_ib_qp *qp, void **seg, int *size, - void **cur_edge) -{ - const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr); - struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr); - u32 pdn = get_pd(qp)->pdn; - u32 xlt_size; - int region_len, ret; - - if (unlikely(wr->wr.num_sge != 1) || - unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || - unlikely(!sig_mr->sig->sig_status_checked)) - return -EINVAL; - - /* length of the protected region, data + protection */ - region_len = wr->wr.sg_list->length; - if (wr->prot && - (wr->prot->lkey != wr->wr.sg_list->lkey || - wr->prot->addr != wr->wr.sg_list->addr || - wr->prot->length != wr->wr.sg_list->length)) - region_len += wr->prot->length; - - /** - * KLM octoword size - if protection was provided - * then we use strided block format (3 octowords), - * else we use single KLM (1 octoword) - **/ - xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm); - - set_sig_umr_segment(*seg, xlt_size); - *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); - *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; - handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - - set_sig_mkey_segment(*seg, wr->sig_mr, wr->access_flags, xlt_size, - region_len, pdn); - *seg += sizeof(struct mlx5_mkey_seg); - *size += sizeof(struct mlx5_mkey_seg) / 16; - handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - - ret = set_sig_data_segment(send_wr, wr->sig_mr, wr->sig_attrs, qp, seg, - size, cur_edge); - if (ret) - return ret; - - sig_mr->sig->sig_status_checked = false; - return 0; -} - static int set_psv_wr(struct ib_sig_domain *domain, u32 psv_idx, void **seg, int *size) { @@ -5187,74 +5121,6 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, num_sge = 0; goto skip_psv; - case IB_WR_REG_SIG_MR: - qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR; - mr = to_mmr(sig_handover_wr(wr)->sig_mr); - - ctrl->imm = cpu_to_be32(mr->ibmr.rkey); - err = set_sig_umr_wr(wr, qp, &seg, &size, - &cur_edge); - if (err) { - mlx5_ib_warn(dev, "\n"); - *bad_wr = wr; - goto out; - } - - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, - MLX5_OPCODE_UMR); - /* - * SET_PSV WQEs are not signaled and solicited - * on error - */ - err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, - &size, &cur_edge, nreq, false, - true); - if (err) { - mlx5_ib_warn(dev, "\n"); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem, - mr->sig->psv_memory.psv_idx, &seg, - &size); - if (err) { - mlx5_ib_warn(dev, "\n"); - *bad_wr = wr; - goto out; - } - - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, - MLX5_OPCODE_SET_PSV); - err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, - &size, &cur_edge, nreq, false, - true); - if (err) { - mlx5_ib_warn(dev, "\n"); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire, - mr->sig->psv_wire.psv_idx, &seg, - &size); - if (err) { - mlx5_ib_warn(dev, "\n"); - *bad_wr = wr; - goto out; - } - - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, - MLX5_OPCODE_SET_PSV); - qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; - num_sge = 0; - goto skip_psv; - default: break; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h index 3c633ab58052..c142f5e7f25f 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h @@ -456,7 +456,7 @@ static inline enum pvrdma_wr_opcode ib_wr_opcode_to_pvrdma(enum ib_wr_opcode op) return PVRDMA_WR_MASKED_ATOMIC_CMP_AND_SWP; case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: return PVRDMA_WR_MASKED_ATOMIC_FETCH_AND_ADD; - case IB_WR_REG_SIG_MR: + case IB_WR_REG_MR_INTEGRITY: return PVRDMA_WR_REG_SIG_MR; default: return PVRDMA_WR_ERROR; -- cgit v1.2.3 From de0ae958deb5e6af35c4c6a4679d4fe9896a98ca Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 11 Jun 2019 18:52:55 +0300 Subject: RDMA/mlx5: Improve PI handover performance In some loads, there is performance degradation when using KLM mkey instead of MTT mkey. This is because KLM descriptor access is via indirection that might require more HW resources and cycles. Using KLM descriptor is not necessary when there are no gaps at the data/metadata sg lists. As an optimization, use MTT mkey whenever it is possible. For that matter, allocate internal MTT mkey and choose the effective pi_mr for in transaction according to the required mapping scheme. The setup of the tested benchmark (using iSER ULP): - 2 servers with 24 cores (1 initiator and 1 target) - ConnectX-4/ConnectX-5 adapters - 24 target sessions with 1 LUN each - ramdisk backstore - PI active Performance results running fio (24 jobs, 128 iodepth) using write_generate=1 and read_verify=1 (w/w.o/baseline): bs IOPS(read) IOPS(write) ---- ---------- ---------- 512 1262.4K/1243.3K/1147.1K 1732.1K/1725.1K/1423.8K 4k 570902/571233/457874 773982/743293/642080 32k 72086/72388/71933 96164/71789/93249 Using write_generate=0 and read_verify=0 (w/w.o patch): bs IOPS(read) IOPS(write) ---- ---------- ---------- 512 1600.1K/1572.1K/1393.3K 1830.3K/1823.5K/1557.2K 4k 937272/921992/762934 815304/753772/646071 32k 77369/75052/72058 97435/73180/94612 Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Suggested-by: Max Gurtovoy Suggested-by: Idan Burstein Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +- drivers/infiniband/hw/mlx5/mr.c | 179 ++++++++++++++++++++++++++++++----- drivers/infiniband/hw/mlx5/qp.c | 2 +- 3 files changed, 164 insertions(+), 24 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index d418219e68c6..405059521321 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -605,7 +605,12 @@ struct mlx5_ib_mr { int access_flags; /* Needed for rereg MR */ struct mlx5_ib_mr *parent; - struct mlx5_ib_mr *pi_mr; /* Needed for IB_MR_TYPE_INTEGRITY */ + /* Needed for IB_MR_TYPE_INTEGRITY */ + struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr *klm_mr; + struct mlx5_ib_mr *mtt_mr; + u64 pi_iova; + atomic_t num_leaf_free; wait_queue_head_t q_leaf_free; struct mlx5_async_work cb_work; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 36d1d6f8bb47..8b40abd0070b 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1641,8 +1641,10 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { struct mlx5_ib_mr *mmr = to_mmr(ibmr); - if (ibmr->type == IB_MR_TYPE_INTEGRITY) - dereg_mr(to_mdev(mmr->pi_mr->ibmr.device), mmr->pi_mr); + if (ibmr->type == IB_MR_TYPE_INTEGRITY) { + dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr); + dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr); + } dereg_mr(to_mdev(ibmr->device), mmr); @@ -1650,7 +1652,8 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, - u32 max_num_sg, u32 max_num_meta_sg) + u32 max_num_sg, u32 max_num_meta_sg, + int desc_size, int access_mode) { struct mlx5_ib_dev *dev = to_mdev(pd->device); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); @@ -1673,16 +1676,17 @@ static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); MLX5_SET(mkc, mkc, free, 1); MLX5_SET(mkc, mkc, translations_octword_size, ndescs); + if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) + MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); - mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; + mr->access_mode = access_mode; - err = mlx5_alloc_priv_descs(pd->device, mr, - ndescs, sizeof(struct mlx5_klm)); + err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); if (err) goto err_free_in; - mr->desc_size = sizeof(struct mlx5_klm); + mr->desc_size = desc_size; mr->max_descs = ndescs; MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); @@ -1786,12 +1790,22 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, mr->sig->sig_err_exists = false; /* Next UMR, Arm SIGERR */ ++mr->sig->sigerr_count; - mr->pi_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, - max_num_meta_sg); - if (IS_ERR(mr->pi_mr)) { - err = PTR_ERR(mr->pi_mr); + mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, + max_num_meta_sg, + sizeof(struct mlx5_klm), + MLX5_MKC_ACCESS_MODE_KLMS); + if (IS_ERR(mr->klm_mr)) { + err = PTR_ERR(mr->klm_mr); goto err_destroy_psv; } + mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, + max_num_meta_sg, + sizeof(struct mlx5_mtt), + MLX5_MKC_ACCESS_MODE_MTT); + if (IS_ERR(mr->mtt_mr)) { + err = PTR_ERR(mr->mtt_mr); + goto err_free_klm_mr; + } } else { mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); err = -EINVAL; @@ -1816,9 +1830,14 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, return &mr->ibmr; err_free_pi_mr: - if (mr->pi_mr) { - dereg_mr(to_mdev(mr->pi_mr->ibmr.device), mr->pi_mr); - mr->pi_mr = NULL; + if (mr->mtt_mr) { + dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); + mr->mtt_mr = NULL; + } +err_free_klm_mr: + if (mr->klm_mr) { + dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); + mr->klm_mr = NULL; } err_destroy_psv: if (mr->sig) { @@ -2056,16 +2075,95 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) return 0; } -int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, +static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + __be64 *descs; + + if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs)) + return -ENOMEM; + + descs = mr->descs; + descs[mr->ndescs + mr->meta_ndescs++] = + cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); + + return 0; +} + +static int +mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, int data_sg_nents, unsigned int *data_sg_offset, struct scatterlist *meta_sg, int meta_sg_nents, unsigned int *meta_sg_offset) { struct mlx5_ib_mr *mr = to_mmr(ibmr); - struct mlx5_ib_mr *pi_mr = mr->pi_mr; + struct mlx5_ib_mr *pi_mr = mr->mtt_mr; int n; + u64 iova; - WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); + pi_mr->ndescs = 0; + pi_mr->meta_ndescs = 0; + pi_mr->meta_length = 0; + + ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + pi_mr->ibmr.page_size = ibmr->page_size; + n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, + mlx5_set_page); + if (n != data_sg_nents) + return n; + + iova = pi_mr->ibmr.iova; + pi_mr->data_length = pi_mr->ibmr.length; + pi_mr->ibmr.length = pi_mr->data_length; + ibmr->length = pi_mr->data_length; + + if (meta_sg_nents) { + u64 page_mask = ~((u64)ibmr->page_size - 1); + + n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, + meta_sg_offset, mlx5_set_page_pi); + + pi_mr->meta_length = pi_mr->ibmr.length; + /* + * PI address for the HW is the offset of the metadata address + * relative to the first data page address. + * It equals to first data page address + size of data pages + + * metadata offset at the first metadata page + */ + pi_mr->pi_iova = (iova & page_mask) + + pi_mr->ndescs * ibmr->page_size + + (pi_mr->ibmr.iova & ~page_mask); + /* + * In order to use one MTT MR for data and metadata, we register + * also the gaps between the end of the data and the start of + * the metadata (the sig MR will verify that the HW will access + * to right addresses). This mapping is safe because we use + * internal mkey for the registration. + */ + pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; + pi_mr->ibmr.iova = iova; + ibmr->length += pi_mr->meta_length; + } + + ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + return n; +} + +static int +mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = mr->klm_mr; + int n; pi_mr->ndescs = 0; pi_mr->meta_ndescs = 0; @@ -2078,19 +2176,56 @@ int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, meta_sg, meta_sg_nents, meta_sg_offset); + ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + /* This is zero-based memory region */ pi_mr->ibmr.iova = 0; + pi_mr->pi_iova = pi_mr->data_length; ibmr->length = pi_mr->ibmr.length; - ibmr->iova = pi_mr->ibmr.iova; - ibmr->sig_attrs->meta_length = pi_mr->meta_length; - ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, - pi_mr->desc_size * pi_mr->max_descs, - DMA_TO_DEVICE); + return n; +} +int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = mr->mtt_mr; + int n; + + WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); + + /* + * As a performance optimization, if possible, there is no need to map + * the sg lists to KLM descriptors. First try to map the sg lists to MTT + * descriptors and fallback to KLM only in case of a failure. + * It's more efficient for the HW to work with MTT descriptors + * (especially in high load). + * Use KLM (indirect access) only if it's mandatory. + */ + n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); + if (n == data_sg_nents + meta_sg_nents) + goto out; + + pi_mr = mr->klm_mr; + n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); if (unlikely(n != data_sg_nents + meta_sg_nents)) return -ENOMEM; +out: + /* This is zero-based memory region */ + ibmr->iova = 0; + mr->pi_mr = pi_mr; + ibmr->sig_attrs->meta_length = pi_mr->meta_length; + return 0; } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 60536c9c008f..f0962be5b11c 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4566,7 +4566,7 @@ static int set_sig_data_segment(const struct ib_send_wr *send_wr, if (pi_mr->meta_ndescs) { prot_len = pi_mr->meta_length; prot_key = pi_mr->ibmr.lkey; - prot_va = pi_mr->ibmr.iova + data_len; + prot_va = pi_mr->pi_iova; prot = true; } -- cgit v1.2.3 From 2563e2f30acb4c914fc475331e476fa920eb4245 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:56 +0300 Subject: RDMA/mlx5: Use PA mapping for PI handover If possibe, avoid doing a UMR operation to register data and protection buffers (via MTT/KLM mkeys). Instead, use the local DMA key and map the SG lists using PA access. This is safe, since the internal key for data and protection never exposed to the remote server (only signature key might be exposed). If PA mappings are not possible, perform mapping using MTT/KLM descriptors. The setup of the tested benchmark (using iSER ULP): - 2 servers with 24 cores (1 initiator and 1 target) - ConnectX-4/ConnectX-5 adapters - 24 target sessions with 1 LUN each - ramdisk backstore - PI active Performance results running fio (24 jobs, 128 iodepth) using write_generate=1 and read_verify=1 (w/w.o patch): bs IOPS(read) IOPS(write) ---- ---------- ---------- 512 1266.4K/1262.4K 1720.1K/1732.1K 4k 793139/570902 1129.6K/773982 32k 72660/72086 97229/96164 Using write_generate=0 and read_verify=0 (w/w.o patch): bs IOPS(read) IOPS(write) ---- ---------- ---------- 512 1590.2K/1600.1K 1828.2K/1830.3K 4k 1078.1K/937272 1142.1K/815304 32k 77012/77369 98125/97435 Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Suggested-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/mr.c | 63 ++++++++++++++++++++++++++-- drivers/infiniband/hw/mlx5/qp.c | 80 ++++++++++++++++++++++++------------ 3 files changed, 114 insertions(+), 30 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 405059521321..bdb83fc85f94 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -609,6 +609,7 @@ struct mlx5_ib_mr { struct mlx5_ib_mr *pi_mr; struct mlx5_ib_mr *klm_mr; struct mlx5_ib_mr *mtt_mr; + u64 data_iova; u64 pi_iova; atomic_t num_leaf_free; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 8b40abd0070b..f2ef89e48afa 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -2001,6 +2001,40 @@ done: return ret; } +static int +mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + unsigned int sg_offset = 0; + int n = 0; + + mr->meta_length = 0; + if (data_sg_nents == 1) { + n++; + mr->ndescs = 1; + if (data_sg_offset) + sg_offset = *data_sg_offset; + mr->data_length = sg_dma_len(data_sg) - sg_offset; + mr->data_iova = sg_dma_address(data_sg) + sg_offset; + if (meta_sg_nents == 1) { + n++; + mr->meta_ndescs = 1; + if (meta_sg_offset) + sg_offset = *meta_sg_offset; + else + sg_offset = 0; + mr->meta_length = sg_dma_len(meta_sg) - sg_offset; + mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; + } + ibmr->length = mr->data_length + mr->meta_length; + } + + return n; +} + static int mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, struct scatterlist *sgl, @@ -2099,7 +2133,6 @@ mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, struct mlx5_ib_mr *mr = to_mmr(ibmr); struct mlx5_ib_mr *pi_mr = mr->mtt_mr; int n; - u64 iova; pi_mr->ndescs = 0; pi_mr->meta_ndescs = 0; @@ -2115,13 +2148,14 @@ mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, if (n != data_sg_nents) return n; - iova = pi_mr->ibmr.iova; + pi_mr->data_iova = pi_mr->ibmr.iova; pi_mr->data_length = pi_mr->ibmr.length; pi_mr->ibmr.length = pi_mr->data_length; ibmr->length = pi_mr->data_length; if (meta_sg_nents) { u64 page_mask = ~((u64)ibmr->page_size - 1); + u64 iova = pi_mr->data_iova; n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, meta_sg_offset, mlx5_set_page_pi); @@ -2181,6 +2215,7 @@ mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, DMA_TO_DEVICE); /* This is zero-based memory region */ + pi_mr->data_iova = 0; pi_mr->ibmr.iova = 0; pi_mr->pi_iova = pi_mr->data_length; ibmr->length = pi_mr->ibmr.length; @@ -2194,11 +2229,27 @@ int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, unsigned int *meta_sg_offset) { struct mlx5_ib_mr *mr = to_mmr(ibmr); - struct mlx5_ib_mr *pi_mr = mr->mtt_mr; + struct mlx5_ib_mr *pi_mr = NULL; int n; WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); + mr->ndescs = 0; + mr->data_length = 0; + mr->data_iova = 0; + mr->meta_ndescs = 0; + mr->pi_iova = 0; + /* + * As a performance optimization, if possible, there is no need to + * perform UMR operation to register the data/metadata buffers. + * First try to map the sg lists to PA descriptors with local_dma_lkey. + * Fallback to UMR only in case of a failure. + */ + n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); + if (n == data_sg_nents + meta_sg_nents) + goto out; /* * As a performance optimization, if possible, there is no need to map * the sg lists to KLM descriptors. First try to map the sg lists to MTT @@ -2207,6 +2258,7 @@ int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, * (especially in high load). * Use KLM (indirect access) only if it's mandatory. */ + pi_mr = mr->mtt_mr; n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, data_sg_offset, meta_sg, meta_sg_nents, meta_sg_offset); @@ -2224,7 +2276,10 @@ out: /* This is zero-based memory region */ ibmr->iova = 0; mr->pi_mr = pi_mr; - ibmr->sig_attrs->meta_length = pi_mr->meta_length; + if (pi_mr) + ibmr->sig_attrs->meta_length = pi_mr->meta_length; + else + ibmr->sig_attrs->meta_length = mr->meta_length; return 0; } diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index f0962be5b11c..4fbf60fed374 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -4562,7 +4562,7 @@ static int set_sig_data_segment(const struct ib_send_wr *send_wr, data_len = pi_mr->data_length; data_key = pi_mr->ibmr.lkey; - data_va = pi_mr->ibmr.iova; + data_va = pi_mr->data_iova; if (pi_mr->meta_ndescs) { prot_len = pi_mr->meta_length; prot_key = pi_mr->ibmr.lkey; @@ -4912,6 +4912,7 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct mlx5_ib_qp *qp; struct mlx5_ib_mr *mr; struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr pa_pi_mr; struct ib_sig_attrs *sig_attrs; struct mlx5_wqe_xrc_seg *xrc; struct mlx5_bf *bf; @@ -5026,35 +5027,62 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, break; case IB_WR_REG_MR_INTEGRITY: - memset(®_pi_wr, 0, sizeof(struct ib_reg_wr)); + qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY; mr = to_mmr(reg_wr(wr)->mr); pi_mr = mr->pi_mr; - reg_pi_wr.mr = &pi_mr->ibmr; - reg_pi_wr.access = reg_wr(wr)->access; - reg_pi_wr.key = pi_mr->ibmr.rkey; - - qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY; - ctrl->imm = cpu_to_be32(reg_pi_wr.key); - /* UMR for data + protection registration */ - err = set_reg_wr(qp, ®_pi_wr, &seg, &size, - &cur_edge, false); - if (err) { - *bad_wr = wr; - goto out; - } - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, - MLX5_OPCODE_UMR); - - err = begin_wqe(qp, &seg, &ctrl, wr, &idx, - &size, &cur_edge, nreq); - if (err) { - mlx5_ib_warn(dev, "\n"); - err = -ENOMEM; - *bad_wr = wr; - goto out; + if (pi_mr) { + memset(®_pi_wr, 0, + sizeof(struct ib_reg_wr)); + + reg_pi_wr.mr = &pi_mr->ibmr; + reg_pi_wr.access = reg_wr(wr)->access; + reg_pi_wr.key = pi_mr->ibmr.rkey; + + ctrl->imm = cpu_to_be32(reg_pi_wr.key); + /* UMR for data + prot registration */ + err = set_reg_wr(qp, ®_pi_wr, &seg, + &size, &cur_edge, + false); + if (err) { + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, + cur_edge, idx, wr->wr_id, + nreq, fence, + MLX5_OPCODE_UMR); + + err = begin_wqe(qp, &seg, &ctrl, wr, + &idx, &size, &cur_edge, + nreq); + if (err) { + mlx5_ib_warn(dev, "\n"); + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + } else { + memset(&pa_pi_mr, 0, + sizeof(struct mlx5_ib_mr)); + /* No UMR, use local_dma_lkey */ + pa_pi_mr.ibmr.lkey = + mr->ibmr.pd->local_dma_lkey; + + pa_pi_mr.ndescs = mr->ndescs; + pa_pi_mr.data_length = mr->data_length; + pa_pi_mr.data_iova = mr->data_iova; + if (mr->meta_ndescs) { + pa_pi_mr.meta_ndescs = + mr->meta_ndescs; + pa_pi_mr.meta_length = + mr->meta_length; + pa_pi_mr.pi_iova = mr->pi_iova; + } + + pa_pi_mr.ibmr.length = mr->ibmr.length; + mr->pi_mr = &pa_pi_mr; } ctrl->imm = cpu_to_be32(mr->ibmr.rkey); /* UMR for sig MR */ -- cgit v1.2.3 From 7796d2a3bb4037f9c51b8d91d059f1d690ed301f Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 11 Jun 2019 18:52:57 +0300 Subject: RDMA/mlx5: Refactor MR descriptors allocation Improve code readability using static helpers for each memory region type. Re-use the common logic to get smaller functions that are easy to maintain and reduce code duplication. Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: Sagi Grimberg Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mr.c | 290 ++++++++++++++++++++++------------------ 1 file changed, 157 insertions(+), 133 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index f2ef89e48afa..6ac77e09a34a 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1651,15 +1651,63 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) return 0; } +static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, + int access_mode, int page_shift) +{ + void *mkc; + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + + MLX5_SET(mkc, mkc, free, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + MLX5_SET(mkc, mkc, translations_octword_size, ndescs); + MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); + MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); + MLX5_SET(mkc, mkc, umr_en, 1); + MLX5_SET(mkc, mkc, log_page_size, page_shift); +} + +static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, int desc_size, int page_shift, + int access_mode, u32 *in, int inlen) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + int err; + + mr->access_mode = access_mode; + mr->desc_size = desc_size; + mr->max_descs = ndescs; + + err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); + if (err) + return err; + + mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); + + err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); + if (err) + goto err_free_descs; + + mr->mmkey.type = MLX5_MKEY_MR; + mr->ibmr.lkey = mr->mmkey.key; + mr->ibmr.rkey = mr->mmkey.key; + + return 0; + +err_free_descs: + mlx5_free_priv_descs(mr); + return err; +} + static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, u32 max_num_sg, u32 max_num_meta_sg, int desc_size, int access_mode) { - struct mlx5_ib_dev *dev = to_mdev(pd->device); int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); + int page_shift = 0; struct mlx5_ib_mr *mr; - void *mkc; u32 *in; int err; @@ -1667,48 +1715,28 @@ static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, if (!mr) return ERR_PTR(-ENOMEM); + mr->ibmr.pd = pd; + mr->ibmr.device = pd->device; + in = kzalloc(inlen, GFP_KERNEL); if (!in) { err = -ENOMEM; goto err_free; } - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, free, 1); - MLX5_SET(mkc, mkc, translations_octword_size, ndescs); if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) - MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + page_shift = PAGE_SHIFT; - mr->access_mode = access_mode; - - err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); + err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, + access_mode, in, inlen); if (err) goto err_free_in; - mr->desc_size = desc_size; - mr->max_descs = ndescs; - - MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); - MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); - MLX5_SET(mkc, mkc, umr_en, 1); - mr->ibmr.pd = pd; - mr->ibmr.device = pd->device; - err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); - if (err) - goto err_priv_descs; - - mr->mmkey.type = MLX5_MKEY_MR; - mr->ibmr.lkey = mr->mmkey.key; - mr->ibmr.rkey = mr->mmkey.key; mr->umem = NULL; kfree(in); return mr; -err_priv_descs: - mlx5_free_priv_descs(mr); err_free_in: kfree(in); err_free: @@ -1716,6 +1744,92 @@ err_free: return ERR_PTR(err); } +static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, u32 *in, int inlen) +{ + return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), + PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in, + inlen); +} + +static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, u32 *in, int inlen) +{ + return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), + 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); +} + +static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int max_num_sg, int max_num_meta_sg, + u32 *in, int inlen) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + u32 psv_index[2]; + void *mkc; + int err; + + mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); + if (!mr->sig) + return -ENOMEM; + + /* create mem & wire PSVs */ + err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); + if (err) + goto err_free_sig; + + mr->sig->psv_memory.psv_idx = psv_index[0]; + mr->sig->psv_wire.psv_idx = psv_index[1]; + + mr->sig->sig_status_checked = true; + mr->sig->sig_err_exists = false; + /* Next UMR, Arm SIGERR */ + ++mr->sig->sigerr_count; + mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, + sizeof(struct mlx5_klm), + MLX5_MKC_ACCESS_MODE_KLMS); + if (IS_ERR(mr->klm_mr)) { + err = PTR_ERR(mr->klm_mr); + goto err_destroy_psv; + } + mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, + sizeof(struct mlx5_mtt), + MLX5_MKC_ACCESS_MODE_MTT); + if (IS_ERR(mr->mtt_mr)) { + err = PTR_ERR(mr->mtt_mr); + goto err_free_klm_mr; + } + + /* Set bsf descriptors for mkey */ + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, bsf_en, 1); + MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); + + err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, + MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); + if (err) + goto err_free_mtt_mr; + + return 0; + +err_free_mtt_mr: + dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); + mr->mtt_mr = NULL; +err_free_klm_mr: + dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); + mr->klm_mr = NULL; +err_destroy_psv: + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) + mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", + mr->sig->psv_memory.psv_idx); + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) + mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", + mr->sig->psv_wire.psv_idx); +err_free_sig: + kfree(mr->sig); + + return err; +} + static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, u32 max_num_meta_sg) @@ -1724,7 +1838,6 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); int ndescs = ALIGN(max_num_sg, 4); struct mlx5_ib_mr *mr; - void *mkc; u32 *in; int err; @@ -1738,121 +1851,32 @@ static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, goto err_free; } - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, free, 1); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); - - if (mr_type == IB_MR_TYPE_MEM_REG) { - mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; - MLX5_SET(mkc, mkc, translations_octword_size, ndescs); - MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - err = mlx5_alloc_priv_descs(pd->device, mr, - ndescs, sizeof(struct mlx5_mtt)); - if (err) - goto err_free_in; - - mr->desc_size = sizeof(struct mlx5_mtt); - mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_SG_GAPS) { - mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; - MLX5_SET(mkc, mkc, translations_octword_size, ndescs); - - err = mlx5_alloc_priv_descs(pd->device, mr, - ndescs, sizeof(struct mlx5_klm)); - if (err) - goto err_free_in; - mr->desc_size = sizeof(struct mlx5_klm); - mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_INTEGRITY) { - u32 psv_index[2]; - - MLX5_SET(mkc, mkc, bsf_en, 1); - MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); - MLX5_SET(mkc, mkc, translations_octword_size, 4); - mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); - if (!mr->sig) { - err = -ENOMEM; - goto err_free_in; - } + mr->ibmr.device = pd->device; + mr->umem = NULL; - /* create mem & wire PSVs */ - err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, - 2, psv_index); - if (err) - goto err_free_sig; - - mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; - mr->sig->psv_memory.psv_idx = psv_index[0]; - mr->sig->psv_wire.psv_idx = psv_index[1]; - - mr->sig->sig_status_checked = true; - mr->sig->sig_err_exists = false; - /* Next UMR, Arm SIGERR */ - ++mr->sig->sigerr_count; - mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, - max_num_meta_sg, - sizeof(struct mlx5_klm), - MLX5_MKC_ACCESS_MODE_KLMS); - if (IS_ERR(mr->klm_mr)) { - err = PTR_ERR(mr->klm_mr); - goto err_destroy_psv; - } - mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, - max_num_meta_sg, - sizeof(struct mlx5_mtt), - MLX5_MKC_ACCESS_MODE_MTT); - if (IS_ERR(mr->mtt_mr)) { - err = PTR_ERR(mr->mtt_mr); - goto err_free_klm_mr; - } - } else { + switch (mr_type) { + case IB_MR_TYPE_MEM_REG: + err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); + break; + case IB_MR_TYPE_SG_GAPS: + err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); + break; + case IB_MR_TYPE_INTEGRITY: + err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, + max_num_meta_sg, in, inlen); + break; + default: mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); err = -EINVAL; - goto err_free_in; } - MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); - MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); - MLX5_SET(mkc, mkc, umr_en, 1); - - mr->ibmr.device = pd->device; - err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); if (err) - goto err_free_pi_mr; + goto err_free_in; - mr->mmkey.type = MLX5_MKEY_MR; - mr->ibmr.lkey = mr->mmkey.key; - mr->ibmr.rkey = mr->mmkey.key; - mr->umem = NULL; kfree(in); return &mr->ibmr; -err_free_pi_mr: - if (mr->mtt_mr) { - dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); - mr->mtt_mr = NULL; - } -err_free_klm_mr: - if (mr->klm_mr) { - dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); - mr->klm_mr = NULL; - } -err_destroy_psv: - if (mr->sig) { - if (mlx5_core_destroy_psv(dev->mdev, - mr->sig->psv_memory.psv_idx)) - mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", - mr->sig->psv_memory.psv_idx); - if (mlx5_core_destroy_psv(dev->mdev, - mr->sig->psv_wire.psv_idx)) - mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", - mr->sig->psv_wire.psv_idx); - } - mlx5_free_priv_descs(mr); -err_free_sig: - kfree(mr->sig); err_free_in: kfree(in); err_free: -- cgit v1.2.3 From 7ef7587541d49c6de1c9650c3c819a7fdd7ade66 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 24 Jun 2019 22:46:08 +0100 Subject: RDMA/hns: fix potential integer overflow on left shift There is a potential integer overflow when int i is left shifted as this is evaluated using 32 bit arithmetic but is being used in a context that expects an expression of type dma_addr_t. Fix this by casting integer i to dma_addr_t before shifting to avoid the overflow. Addresses-Coverity: ("Unintentional integer overflow") Fixes: 2ac0bc5e725e ("RDMA/hns: Add a group interfaces for optimizing buffers getting flow") Signed-off-by: Colin Ian King Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 14fcc359599c..2c8defa94107 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -257,7 +257,7 @@ int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, for (i = start; i < end; i++) if (buf->nbufs == 1) bufs[total++] = buf->direct.map + - (i << buf->page_shift); + ((dma_addr_t)i << buf->page_shift); else bufs[total++] = buf->page_list[i].map; -- cgit v1.2.3 From b417c0879db72f810ca81d88b719e70d20566857 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Sat, 8 Jun 2019 12:27:14 +0300 Subject: RDMA/hns: Fix an error code in hns_roce_set_user_sq_size() This function is supposed to return negative kernel error codes but here it returns CMD_RST_PRC_EBUSY (2). The error code eventually gets passed to IS_ERR() and since it's not an error pointer it leads to an Oops in hns_roce_v1_rsv_lp_qp() Signed-off-by: Dan Carpenter Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index edd62b4dc0a0..7f2da5e10e67 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1098,7 +1098,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, if (ret == CMD_RST_PRC_SUCCESS) return 0; if (ret == CMD_RST_PRC_EBUSY) - return ret; + return -EBUSY; ret = __hns_roce_cmq_send(hr_dev, desc, num); if (ret) { @@ -1106,7 +1106,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, if (retval == CMD_RST_PRC_SUCCESS) return 0; else if (retval == CMD_RST_PRC_EBUSY) - return retval; + return -EBUSY; } return ret; -- cgit v1.2.3 From e9816ddf2a33f3afdf3dfc35c21aafad389ee482 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Wed, 19 Jun 2019 15:00:47 +0800 Subject: RDMA/hns: Cleanup unnecessary exported symbols This patch removes the hns-roce.ko for cleanup all the exported symbols in common part. Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/Kconfig | 15 +++++++-------- drivers/infiniband/hw/hns/Makefile | 13 +++++++++---- drivers/infiniband/hw/hns/hns_roce_alloc.c | 2 -- drivers/infiniband/hw/hns/hns_roce_cmd.c | 4 ---- drivers/infiniband/hw/hns/hns_roce_cq.c | 5 ----- drivers/infiniband/hw/hns/hns_roce_db.c | 4 ---- drivers/infiniband/hw/hns/hns_roce_hem.c | 5 ----- drivers/infiniband/hw/hns/hns_roce_main.c | 3 --- drivers/infiniband/hw/hns/hns_roce_mr.c | 5 ----- drivers/infiniband/hw/hns/hns_roce_pd.c | 2 -- drivers/infiniband/hw/hns/hns_roce_qp.c | 13 ------------- drivers/infiniband/hw/hns/hns_roce_srq.c | 1 - 12 files changed, 16 insertions(+), 56 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig index fddb5fdf92de..b548c003009b 100644 --- a/drivers/infiniband/hw/hns/Kconfig +++ b/drivers/infiniband/hw/hns/Kconfig @@ -7,25 +7,24 @@ config INFINIBAND_HNS is used in Hisilicon Hip06 and more further ICT SoC based on platform device. - To compile this driver as a module, choose M here: the module - will be called hns-roce. + To compile HIP06 or HIP08 driver as module, choose M here. config INFINIBAND_HNS_HIP06 - tristate "Hisilicon Hip06 Family RoCE support" + bool "Hisilicon Hip06 Family RoCE support" depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET ---help--- RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and Hip07 SoC. These RoCE engines are platform devices. - To compile this driver as a module, choose M here: the module - will be called hns-roce-hw-v1. + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v1 config INFINIBAND_HNS_HIP08 - tristate "Hisilicon Hip08 Family RoCE support" + bool "Hisilicon Hip08 Family RoCE support" depends on INFINIBAND_HNS && PCI && HNS3 ---help--- RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC. The RoCE engine is a PCI device. - To compile this driver as a module, choose M here: the module - will be called hns-roce-hw-v2. + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v2. diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index eee5205f936f..b956cf430bdb 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -4,11 +4,16 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 -obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o -obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o + +ifdef CONFIG_INFINIBAND_HNS_HIP06 hns-roce-hw-v1-objs := hns_roce_hw_v1.o -obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o -hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o $(hns-roce-objs) +endif + +ifdef CONFIG_INFINIBAND_HNS_HIP08 +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o +endif diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index 2c8defa94107..8c063c598d2a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -68,7 +68,6 @@ void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, { hns_roce_bitmap_free_range(bitmap, obj, 1, rr); } -EXPORT_SYMBOL_GPL(hns_roce_bitmap_free); int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, int align, unsigned long *obj) @@ -175,7 +174,6 @@ void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, kfree(buf->page_list); } } -EXPORT_SYMBOL_GPL(hns_roce_buf_free); int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, struct hns_roce_buf *buf, u32 page_shift) diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index 2acf946d02e5..b83d5bd92329 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -103,7 +103,6 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, context->out_param = out_param; complete(&context->done); } -EXPORT_SYMBOL_GPL(hns_roce_cmd_event); /* this should be called with "use_events" */ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, @@ -204,7 +203,6 @@ int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, return ret; } -EXPORT_SYMBOL_GPL(hns_roce_cmd_mbox); int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) { @@ -291,7 +289,6 @@ struct hns_roce_cmd_mailbox return mailbox; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_cmd_mailbox); void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox) @@ -302,4 +299,3 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } -EXPORT_SYMBOL_GPL(hns_roce_free_cmd_mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 6b4d8e50aabe..4e50c22a2da4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -205,7 +205,6 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); } -EXPORT_SYMBOL_GPL(hns_roce_free_cq); static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, struct ib_udata *udata, @@ -435,7 +434,6 @@ err_db: err_cq: return ret; } -EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq); void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { @@ -465,7 +463,6 @@ void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) hns_roce_free_db(hr_dev, &hr_cq->db); } } -EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq); void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) { @@ -481,7 +478,6 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) ++cq->arm_sn; cq->comp(cq); } -EXPORT_SYMBOL_GPL(hns_roce_cq_completion); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) { @@ -503,7 +499,6 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); } -EXPORT_SYMBOL_GPL(hns_roce_cq_event); int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) { diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index 3a040a9390d8..627aa46ef683 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -51,7 +51,6 @@ out: return ret; } -EXPORT_SYMBOL(hns_roce_db_map_user); void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, struct hns_roce_db *db) @@ -67,7 +66,6 @@ void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, mutex_unlock(&context->page_mutex); } -EXPORT_SYMBOL(hns_roce_db_unmap_user); static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( struct device *dma_device) @@ -151,7 +149,6 @@ out: return ret; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_db); void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) { @@ -181,4 +178,3 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) mutex_unlock(&hr_dev->pgdir_mutex); } -EXPORT_SYMBOL_GPL(hns_roce_free_db); diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index d145e3ed21d4..f4da5bd2884f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -56,7 +56,6 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) return false; } -EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, u32 bt_chunk_num) @@ -234,7 +233,6 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, return 0; } -EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages, @@ -621,7 +619,6 @@ out: mutex_unlock(&table->mutex); return ret; } -EXPORT_SYMBOL_GPL(hns_roce_table_get); static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -764,7 +761,6 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev, mutex_unlock(&table->mutex); } -EXPORT_SYMBOL_GPL(hns_roce_table_put); void *hns_roce_table_find(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -837,7 +833,6 @@ out: mutex_unlock(&table->mutex); return addr; } -EXPORT_SYMBOL_GPL(hns_roce_table_find); int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index c0e819ed8c9b..9f83acec6001 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -57,7 +57,6 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) { return gid_index * hr_dev->caps.num_ports + port; } -EXPORT_SYMBOL_GPL(hns_get_gid_index); static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) { @@ -974,7 +973,6 @@ error_failed_cmq_init: return ret; } -EXPORT_SYMBOL_GPL(hns_roce_init); void hns_roce_exit(struct hns_roce_dev *hr_dev) { @@ -995,7 +993,6 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) if (hr_dev->hw->reset) hr_dev->hw->reset(hr_dev, false); } -EXPORT_SYMBOL_GPL(hns_roce_exit); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Wei Hu "); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index adf075183dfb..549e1a38dfe0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -47,7 +47,6 @@ unsigned long key_to_hw_index(u32 key) { return (key << 24) | (key >> 8); } -EXPORT_SYMBOL_GPL(key_to_hw_index); static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox, @@ -66,7 +65,6 @@ int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, HNS_ROCE_CMD_TIMEOUT_MSECS); } -EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt); static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, unsigned long *seg) @@ -293,7 +291,6 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) break; } } -EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup); static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, int err_loop_index, @@ -1507,7 +1504,6 @@ void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, { hns_roce_hem_list_release(hr_dev, &mtr->hem_list); } -EXPORT_SYMBOL_GPL(hns_roce_mtr_cleanup); static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, dma_addr_t *bufs, @@ -1611,4 +1607,3 @@ done: return total; } -EXPORT_SYMBOL_GPL(hns_roce_mtr_find); diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 813401384d78..920ca76b5db1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -83,13 +83,11 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_pd); void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn); } -EXPORT_SYMBOL_GPL(hns_roce_dealloc_pd); int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) { diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 7e9db8236072..ee7e1fef31e7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -64,7 +64,6 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); } -EXPORT_SYMBOL_GPL(hns_roce_qp_event); static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, enum hns_roce_event type) @@ -139,7 +138,6 @@ enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) return HNS_ROCE_QP_NUM_STATE; } } -EXPORT_SYMBOL_GPL(to_hns_roce_state); static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, struct hns_roce_qp *hr_qp) @@ -242,7 +240,6 @@ void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); xa_unlock_irqrestore(xa, flags); } -EXPORT_SYMBOL_GPL(hns_roce_qp_remove); void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { @@ -260,7 +257,6 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); } } -EXPORT_SYMBOL_GPL(hns_roce_qp_free); void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, int cnt) @@ -272,7 +268,6 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); } -EXPORT_SYMBOL_GPL(hns_roce_release_range_qp); static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, bool is_user, int has_rq, @@ -1027,7 +1022,6 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, return &hr_qp->ibqp; } -EXPORT_SYMBOL_GPL(hns_roce_create_qp); int to_hr_qp_type(int qp_type) { @@ -1046,7 +1040,6 @@ int to_hr_qp_type(int qp_type) return transport_type; } -EXPORT_SYMBOL_GPL(to_hr_qp_type); int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) @@ -1166,7 +1159,6 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); } } -EXPORT_SYMBOL_GPL(hns_roce_lock_cqs); void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) @@ -1183,7 +1175,6 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, spin_unlock_irq(&recv_cq->lock); } } -EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs); static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) { @@ -1195,20 +1186,17 @@ void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); } -EXPORT_SYMBOL_GPL(get_recv_wqe); void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); } -EXPORT_SYMBOL_GPL(get_send_wqe); void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) { return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); } -EXPORT_SYMBOL_GPL(get_send_extend_sge); bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, struct ib_cq *ib_cq) @@ -1227,7 +1215,6 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, return cur + nreq >= hr_wq->max_post; } -EXPORT_SYMBOL_GPL(hns_roce_wq_overflow); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) { diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index de645be8aa48..38bb548eaa6d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -30,7 +30,6 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) if (atomic_dec_and_test(&srq->refcount)) complete(&srq->free); } -EXPORT_SYMBOL_GPL(hns_roce_srq_event); static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, enum hns_roce_event event_type) -- cgit v1.2.3 From 10dcc7448e9ea49488a38bca7551de1a9da06ad9 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 24 Jun 2019 13:16:49 +0100 Subject: RDMA/hns: fix spelling mistake "attatch" -> "attach" There is a spelling mistake in an dev_err message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_qp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index ee7e1fef31e7..4f693cded74e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -853,7 +853,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, hr_qp->regions, hr_qp->region_cnt); if (ret) { - dev_err(dev, "mtr attatch error for create qp\n"); + dev_err(dev, "mtr attach error for create qp\n"); goto err_mtr; } -- cgit v1.2.3 From 239b0e52d8aa64d2559c672fd8c29cf1fffc3ec7 Mon Sep 17 00:00:00 2001 From: Kamenee Arumugam Date: Fri, 28 Jun 2019 14:04:17 -0400 Subject: IB/hfi1: Move rvt_cq_wc struct into uapi directory The rvt_cq_wc struct elements are shared between rdmavt and the providers but not in uapi directory. As per the comment in https://marc.info/?l=linux-rdma&m=152296522708522&w=2 The hfi1 driver and the rdma core driver are not using shared structures in the uapi directory. In that case, move rvt_cq_wc struct into the rvt-abi.h header file and create a rvt_k_cq_w for the kernel completion queue. Signed-off-by: Kamenee Arumugam Reviewed-by: Mike Marciniszyn Signed-off-by: Mike Marciniszyn Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/qp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index 4e0e9fc0a777..41261e72c429 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -702,8 +702,8 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) sde ? sde->this_idx : 0, send_context, send_context ? send_context->sw_index : 0, - ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head, - ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail, + ib_cq_head(qp->ibqp.send_cq), + ib_cq_tail(qp->ibqp.send_cq), qp->pid, qp->s_state, qp->s_ack_state, -- cgit v1.2.3 From 5136bfea7e79b333af77594fac5bc70282a95313 Mon Sep 17 00:00:00 2001 From: Kamenee Arumugam Date: Fri, 28 Jun 2019 14:21:52 -0400 Subject: IB/{hfi1, qib, rdmavt}: Put qp in error state when cq is full When a completion queue is full, the associated queue pairs are not put into the error state. According to the IBTA specification, this is a violation. Quote from IBTA spec: C9-218: A Requester Class F error occurs when the CQ is inaccessible or full and an attempt is made to complete a WQE. The Affected QP shall be moved to the error state and affiliated asynchronous errors generated as described in 11.6.3.1 Affiliated Asynchronous Events on page 678. The current WQE and any subsequent WQEs are left in an unknown state. C11-37: The CI shall generate a CQ Error when a CQ overrun is detected. This condition will result in an Affiliated Asynchronous Error for any associated Work Queues when they attempt to use that CQ. Completions can no longer be added to the CQ. It is not guaranteed that completions present in the CQ at the time the error occurred can be retrieved. Possible causes include a CQ overrun or a CQ protection error. Put the qp in error state when cq is full. Implement a state called full to continue to put other associated QPs in error state. Reviewed-by: Mike Marciniszyn Reviewed-by: Michael J. Ruhl Signed-off-by: Kamenee Arumugam Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/rc.c | 3 +-- drivers/infiniband/hw/hfi1/uc.c | 3 +-- drivers/infiniband/hw/hfi1/ud.c | 5 ++--- drivers/infiniband/hw/qib/qib_rc.c | 3 +-- drivers/infiniband/hw/qib/qib_uc.c | 3 +-- drivers/infiniband/hw/qib/qib_ud.c | 6 ++---- 6 files changed, 8 insertions(+), 15 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index 235bdbc706ac..0477c14633ab 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -3008,8 +3008,7 @@ send_last: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_ONLY): diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c index 4ed4fcfabd6c..0c77f18120ed 100644 --- a/drivers/infiniband/hw/hfi1/uc.c +++ b/drivers/infiniband/hw/hfi1/uc.c @@ -476,8 +476,7 @@ last_imm: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index 4cb0fce5c096..e16d499cfd1e 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -255,8 +255,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - swqe->wr.send_flags & IB_SEND_SOLICITED); + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); ibp->rvp.n_loop_pkts++; bail_unlock: spin_unlock_irqrestore(&qp->r_lock, flags); @@ -1061,7 +1060,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, solicited); + rvt_recv_cq(qp, &wc, solicited); return; drop: diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index 8d9a94d6f685..1d5e2d4ee257 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -1891,8 +1891,7 @@ send_last: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c index 30c70ad0f4bf..e17b91e2c22a 100644 --- a/drivers/infiniband/hw/qib/qib_uc.c +++ b/drivers/infiniband/hw/qib/qib_uc.c @@ -400,8 +400,7 @@ last_imm: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index 5cdedba2d164..32ad0b635fc6 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -210,8 +210,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - swqe->wr.send_flags & IB_SEND_SOLICITED); + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); ibp->rvp.n_loop_pkts++; bail_unlock: spin_unlock_irqrestore(&qp->r_lock, flags); @@ -573,8 +572,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr, dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); return; drop: -- cgit v1.2.3 From d310c4bf8aeacc0256091feb6a0337b8fef763ac Mon Sep 17 00:00:00 2001 From: "Michael J. Ruhl" Date: Fri, 28 Jun 2019 14:22:04 -0400 Subject: IB/{rdmavt, hfi1, qib}: Remove AH refcount for UD QPs Historically rdmavt destroy_ah() has returned an -EBUSY when the AH has a non-zero reference count. IBTA 11.2.2 notes no such return value or error case: Output Modifiers: - Verb results: - Operation completed successfully. - Invalid HCA handle. - Invalid address handle. ULPs never test for this error and this will leak memory. The reference count exists to allow for driver independent progress mechanisms to process UD SWQEs in parallel with post sends. The SWQE will hold a reference count until the UD SWQE completes and then drops the reference. Fix by removing need to reference count the AH. Add a UD specific allocation to each SWQE entry to cache the necessary information for independent progress. Copy the information during the post send processing. Reviewed-by: Mike Marciniszyn Signed-off-by: Mike Marciniszyn Signed-off-by: Michael J. Ruhl Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/qp.c | 4 ++-- drivers/infiniband/hw/hfi1/ud.c | 30 +++++++++++++++--------------- drivers/infiniband/hw/qib/qib_qp.c | 4 ++-- drivers/infiniband/hw/qib/qib_ud.c | 21 +++++++++++---------- 4 files changed, 30 insertions(+), 29 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index 41261e72c429..a84b44af7b97 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -348,7 +348,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) break; case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.ah); + ah = ibah_to_rvtah(wqe->ud_wr.wr.ah); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf) diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index e16d499cfd1e..f8e796e45517 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -87,7 +87,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) rcu_read_lock(); qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, - swqe->ud_wr.remote_qpn); + swqe->ud_wr.wr.remote_qpn); if (!qp) { ibp->rvp.n_pkt_drops++; rcu_read_unlock(); @@ -105,7 +105,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; + ah_attr = swqe->ud_wr.attr; ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -135,8 +135,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.remote_qkey; + qkey = (int)swqe->ud_wr.wr.remote_qkey < 0 ? + sqp->qkey : swqe->ud_wr.wr.remote_qkey; if (unlikely(qkey != qp->qkey)) goto drop; /* silently drop per IBTA spec */ } @@ -240,7 +240,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { if (sqp->ibqp.qp_type == IB_QPT_GSI || sqp->ibqp.qp_type == IB_QPT_SMI) - wc.pkey_index = swqe->ud_wr.pkey_index; + wc.pkey_index = swqe->ud_wr.wr.pkey_index; else wc.pkey_index = sqp->s_pkey_index; } else { @@ -282,20 +282,20 @@ static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, bth0 |= IB_BTH_SOLICITED; bth0 |= extra_bytes << 20; if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) - *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); + *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.wr.pkey_index); else *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); if (!bypass) bth0 |= *pkey; ohdr->bth[0] = cpu_to_be32(bth0); - ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn); + ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.wr.remote_qpn); ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.remote_qkey); + ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.wr.remote_qkey < 0 ? + qp->qkey : wqe->ud_wr.wr.remote_qkey); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); } @@ -315,7 +315,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = wqe->ud_wr.attr; extra_bytes = -wqe->length & 3; nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; @@ -379,7 +379,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, struct hfi1_pportdata *ppd; struct hfi1_ibport *ibp; u32 dlid, slid, nwords, extra_bytes; - u32 dest_qp = wqe->ud_wr.remote_qpn; + u32 dest_qp = wqe->ud_wr.wr.remote_qpn; u32 src_qp = qp->ibqp.qp_num; u16 len, pkey; u8 l4, sc5; @@ -387,7 +387,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = wqe->ud_wr.attr; /* * Build 16B Management Packet if either the destination @@ -449,7 +449,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, if (is_mgmt) { l4 = OPA_16B_L4_FM; - pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); + pkey = hfi1_get_pkey(ibp, wqe->ud_wr.wr.pkey_index); hfi1_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt, dest_qp, src_qp); } else { @@ -514,7 +514,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = wqe->ud_wr.attr; priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr); if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) || (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) { diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index a81905df2d0f..0e1d0d692891 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 - 2017 Intel Corporation. All rights reserved. + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. * Copyright (c) 2006 - 2012 QLogic Corporation. * All rights reserved. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. * @@ -398,7 +398,7 @@ int qib_check_send_wqe(struct rvt_qp *qp, case IB_QPT_SMI: case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.ah); + ah = ibah_to_rvtah(wqe->ud_wr.wr.ah); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; /* progress hint */ diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index 32ad0b635fc6..d8c2c968909f 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. * @@ -63,7 +64,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) enum ib_qp_type sqptype, dqptype; rcu_read_lock(); - qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.remote_qpn); + qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.wr.remote_qpn); if (!qp) { ibp->rvp.n_pkt_drops++; goto drop; @@ -80,7 +81,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; + ah_attr = swqe->ud_wr.attr; ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -110,8 +111,8 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.remote_qkey; + qkey = (int)swqe->ud_wr.wr.remote_qkey < 0 ? + sqp->qkey : swqe->ud_wr.wr.remote_qkey; if (unlikely(qkey != qp->qkey)) goto drop; } @@ -203,7 +204,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.qp = &qp->ibqp; wc.src_qp = sqp->ibqp.qp_num; wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? - swqe->ud_wr.pkey_index : 0; + swqe->ud_wr.wr.pkey_index : 0; wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & ((1 << ppd->lmc) - 1)); wc.sl = rdma_ah_get_sl(ah_attr); @@ -270,7 +271,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = wqe->ud_wr.attr; if (rdma_ah_get_dlid(ah_attr) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) { if (rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE)) @@ -362,7 +363,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) bth0 |= extra_bytes << 20; bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? - wqe->ud_wr.pkey_index : qp->s_pkey_index); + wqe->ud_wr.wr.pkey_index : qp->s_pkey_index); ohdr->bth[0] = cpu_to_be32(bth0); /* * Use the multicast QP if the destination LID is a multicast LID. @@ -371,14 +372,14 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) be16_to_cpu(IB_MULTICAST_LID_BASE) && rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE) ? cpu_to_be32(QIB_MULTICAST_QPN) : - cpu_to_be32(wqe->ud_wr.remote_qpn); + cpu_to_be32(wqe->ud_wr.wr.remote_qpn); ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.remote_qkey); + ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.wr.remote_qkey < 0 ? + qp->qkey : wqe->ud_wr.wr.remote_qkey); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); done: -- cgit v1.2.3 From 2b0ad2da8fd4c32f63d9142f2de43a4d34fdd679 Mon Sep 17 00:00:00 2001 From: "Michael J. Ruhl" Date: Fri, 28 Jun 2019 14:22:11 -0400 Subject: IB/{rdmavt, hfi1, qib}: Add helpers to hide SWQE WR details Add some helper functions to hide struct rvt_swqe details. Reviewed-by: Mike Marciniszyn Signed-off-by: Michael J. Ruhl Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/qp.c | 2 +- drivers/infiniband/hw/hfi1/ud.c | 29 +++++++++++++++-------------- drivers/infiniband/hw/qib/qib_qp.c | 2 +- drivers/infiniband/hw/qib/qib_ud.c | 21 +++++++++++---------- 4 files changed, 28 insertions(+), 26 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index a84b44af7b97..f8e733aa3bb8 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -348,7 +348,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) break; case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.wr.ah); + ah = rvt_get_swqe_ah(wqe); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf) diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index f8e796e45517..e804af71b629 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -87,7 +87,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) rcu_read_lock(); qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, - swqe->ud_wr.wr.remote_qpn); + rvt_get_swqe_remote_qpn(swqe)); if (!qp) { ibp->rvp.n_pkt_drops++; rcu_read_unlock(); @@ -105,7 +105,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = swqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(swqe); ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -135,8 +135,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.wr.remote_qkey; + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); if (unlikely(qkey != qp->qkey)) goto drop; /* silently drop per IBTA spec */ } @@ -240,7 +240,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { if (sqp->ibqp.qp_type == IB_QPT_GSI || sqp->ibqp.qp_type == IB_QPT_SMI) - wc.pkey_index = swqe->ud_wr.wr.pkey_index; + wc.pkey_index = rvt_get_swqe_pkey_index(swqe); else wc.pkey_index = sqp->s_pkey_index; } else { @@ -282,20 +282,21 @@ static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, bth0 |= IB_BTH_SOLICITED; bth0 |= extra_bytes << 20; if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) - *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.wr.pkey_index); + *pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); else *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); if (!bypass) bth0 |= *pkey; ohdr->bth[0] = cpu_to_be32(bth0); - ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.wr.remote_qpn); + ohdr->bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.wr.remote_qkey); + ohdr->u.ud.deth[0] = + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : + rvt_get_swqe_remote_qkey(wqe)); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); } @@ -315,7 +316,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = wqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); extra_bytes = -wqe->length & 3; nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; @@ -379,7 +380,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, struct hfi1_pportdata *ppd; struct hfi1_ibport *ibp; u32 dlid, slid, nwords, extra_bytes; - u32 dest_qp = wqe->ud_wr.wr.remote_qpn; + u32 dest_qp = rvt_get_swqe_remote_qpn(wqe); u32 src_qp = qp->ibqp.qp_num; u16 len, pkey; u8 l4, sc5; @@ -387,7 +388,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = wqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); /* * Build 16B Management Packet if either the destination @@ -449,7 +450,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, if (is_mgmt) { l4 = OPA_16B_L4_FM; - pkey = hfi1_get_pkey(ibp, wqe->ud_wr.wr.pkey_index); + pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); hfi1_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt, dest_qp, src_qp); } else { @@ -514,7 +515,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = wqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr); if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) || (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) { diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index 0e1d0d692891..8d0563ef5be1 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -398,7 +398,7 @@ int qib_check_send_wqe(struct rvt_qp *qp, case IB_QPT_SMI: case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.wr.ah); + ah = rvt_get_swqe_ah(wqe); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; /* progress hint */ diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index d8c2c968909f..93ca21347959 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -64,7 +64,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) enum ib_qp_type sqptype, dqptype; rcu_read_lock(); - qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.wr.remote_qpn); + qp = rvt_lookup_qpn(rdi, &ibp->rvp, rvt_get_swqe_remote_qpn(swqe)); if (!qp) { ibp->rvp.n_pkt_drops++; goto drop; @@ -81,7 +81,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = swqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(swqe); ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -111,8 +111,8 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.wr.remote_qkey; + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); if (unlikely(qkey != qp->qkey)) goto drop; } @@ -204,7 +204,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.qp = &qp->ibqp; wc.src_qp = sqp->ibqp.qp_num; wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? - swqe->ud_wr.wr.pkey_index : 0; + rvt_get_swqe_pkey_index(swqe) : 0; wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & ((1 << ppd->lmc) - 1)); wc.sl = rdma_ah_get_sl(ah_attr); @@ -271,7 +271,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = wqe->ud_wr.attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); if (rdma_ah_get_dlid(ah_attr) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) { if (rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE)) @@ -363,7 +363,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) bth0 |= extra_bytes << 20; bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? - wqe->ud_wr.wr.pkey_index : qp->s_pkey_index); + rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); ohdr->bth[0] = cpu_to_be32(bth0); /* * Use the multicast QP if the destination LID is a multicast LID. @@ -372,14 +372,15 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) be16_to_cpu(IB_MULTICAST_LID_BASE) && rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE) ? cpu_to_be32(QIB_MULTICAST_QPN) : - cpu_to_be32(wqe->ud_wr.wr.remote_qpn); + cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.wr.remote_qkey); + ohdr->u.ud.deth[0] = + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : + rvt_get_swqe_remote_qkey(wqe)); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); done: -- cgit v1.2.3 From bf3b1e0ce093ce31d4d91d613f9b09d80a4021cc Mon Sep 17 00:00:00 2001 From: "Michael J. Ruhl" Date: Fri, 28 Jun 2019 14:22:17 -0400 Subject: IB/hfi1: Reduce excessive aspm inlines Uninline the aspm API since it increases code space for no reason. Move the aspm module param to the new aspm C file. Reviewed-by: Mike Marciniszyn Signed-off-by: Michael J. Ruhl Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/Makefile | 1 + drivers/infiniband/hw/hfi1/aspm.c | 270 ++++++++++++++++++++++++++++++++++++ drivers/infiniband/hw/hfi1/aspm.h | 262 ++-------------------------------- drivers/infiniband/hw/hfi1/pcie.c | 6 +- 4 files changed, 280 insertions(+), 259 deletions(-) create mode 100644 drivers/infiniband/hw/hfi1/aspm.c (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile index 4044a8c8dbf4..0405d26d0833 100644 --- a/drivers/infiniband/hw/hfi1/Makefile +++ b/drivers/infiniband/hw/hfi1/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o hfi1-y := \ affinity.o \ + aspm.o \ chip.o \ device.o \ driver.o \ diff --git a/drivers/infiniband/hw/hfi1/aspm.c b/drivers/infiniband/hw/hfi1/aspm.c new file mode 100644 index 000000000000..a3c53be4072c --- /dev/null +++ b/drivers/infiniband/hw/hfi1/aspm.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright(c) 2019 Intel Corporation. + * + */ + +#include "aspm.h" + +/* Time after which the timer interrupt will re-enable ASPM */ +#define ASPM_TIMER_MS 1000 +/* Time for which interrupts are ignored after a timer has been scheduled */ +#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) +/* Two interrupts within this time trigger ASPM disable */ +#define ASPM_TRIGGER_MS 1 +#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) +#define ASPM_L1_SUPPORTED(reg) \ + ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) + +uint aspm_mode = ASPM_MODE_DISABLED; +module_param_named(aspm, aspm_mode, uint, 0444); +MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); + +static bool aspm_hw_l1_supported(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + u32 up, dn; + + /* + * If the driver does not have access to the upstream component, + * it cannot support ASPM L1 at all. + */ + if (!parent) + return false; + + pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); + dn = ASPM_L1_SUPPORTED(dn); + + pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); + up = ASPM_L1_SUPPORTED(up); + + /* ASPM works on A-step but is reported as not supported */ + return (!!dn || is_ax(dd)) && !!up; +} + +/* Set L1 entrance latency for slower entry to L1 */ +static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) +{ + u32 l1_ent_lat = 0x4u; + u32 reg32; + + pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); + reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; + reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; + pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); +} + +static void aspm_hw_enable_l1(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + + /* + * If the driver does not have access to the upstream component, + * it cannot support ASPM L1 at all. + */ + if (!parent) + return; + + /* Enable ASPM L1 first in upstream component and then downstream */ + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + PCI_EXP_LNKCTL_ASPM_L1); + pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + PCI_EXP_LNKCTL_ASPM_L1); +} + +void aspm_hw_disable_l1(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + + /* Disable ASPM L1 first in downstream component and then upstream */ + pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0x0); + if (parent) + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0x0); +} + +static void aspm_enable(struct hfi1_devdata *dd) +{ + if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || + !dd->aspm_supported) + return; + + aspm_hw_enable_l1(dd); + dd->aspm_enabled = true; +} + +static void aspm_disable(struct hfi1_devdata *dd) +{ + if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) + return; + + aspm_hw_disable_l1(dd); + dd->aspm_enabled = false; +} + +static void aspm_disable_inc(struct hfi1_devdata *dd) +{ + unsigned long flags; + + spin_lock_irqsave(&dd->aspm_lock, flags); + aspm_disable(dd); + atomic_inc(&dd->aspm_disabled_cnt); + spin_unlock_irqrestore(&dd->aspm_lock, flags); +} + +static void aspm_enable_dec(struct hfi1_devdata *dd) +{ + unsigned long flags; + + spin_lock_irqsave(&dd->aspm_lock, flags); + if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) + aspm_enable(dd); + spin_unlock_irqrestore(&dd->aspm_lock, flags); +} + +/* ASPM processing for each receive context interrupt */ +void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd) +{ + bool restart_timer; + bool close_interrupts; + unsigned long flags; + ktime_t now, prev; + + spin_lock_irqsave(&rcd->aspm_lock, flags); + /* PSM contexts are open */ + if (!rcd->aspm_intr_enable) + goto unlock; + + prev = rcd->aspm_ts_last_intr; + now = ktime_get(); + rcd->aspm_ts_last_intr = now; + + /* An interrupt pair close together in time */ + close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; + + /* Don't push out our timer till this much time has elapsed */ + restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > + ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; + restart_timer = restart_timer && close_interrupts; + + /* Disable ASPM and schedule timer */ + if (rcd->aspm_enabled && close_interrupts) { + aspm_disable_inc(rcd->dd); + rcd->aspm_enabled = false; + restart_timer = true; + } + + if (restart_timer) { + mod_timer(&rcd->aspm_timer, + jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); + rcd->aspm_ts_timer_sched = now; + } +unlock: + spin_unlock_irqrestore(&rcd->aspm_lock, flags); +} + +/* Timer function for re-enabling ASPM in the absence of interrupt activity */ +static void aspm_ctx_timer_function(struct timer_list *t) +{ + struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); + unsigned long flags; + + spin_lock_irqsave(&rcd->aspm_lock, flags); + aspm_enable_dec(rcd->dd); + rcd->aspm_enabled = true; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); +} + +/* + * Disable interrupt processing for verbs contexts when PSM or VNIC contexts + * are open. + */ +void aspm_disable_all(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + unsigned long flags; + u16 i; + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) { + del_timer_sync(&rcd->aspm_timer); + spin_lock_irqsave(&rcd->aspm_lock, flags); + rcd->aspm_intr_enable = false; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); + hfi1_rcd_put(rcd); + } + } + + aspm_disable(dd); + atomic_set(&dd->aspm_disabled_cnt, 0); +} + +/* Re-enable interrupt processing for verbs contexts */ +void aspm_enable_all(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + unsigned long flags; + u16 i; + + aspm_enable(dd); + + if (aspm_mode != ASPM_MODE_DYNAMIC) + return; + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) { + spin_lock_irqsave(&rcd->aspm_lock, flags); + rcd->aspm_intr_enable = true; + rcd->aspm_enabled = true; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); + hfi1_rcd_put(rcd); + } + } +} + +static void aspm_ctx_init(struct hfi1_ctxtdata *rcd) +{ + spin_lock_init(&rcd->aspm_lock); + timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); + rcd->aspm_intr_supported = rcd->dd->aspm_supported && + aspm_mode == ASPM_MODE_DYNAMIC && + rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; +} + +void aspm_init(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + u16 i; + + spin_lock_init(&dd->aspm_lock); + dd->aspm_supported = aspm_hw_l1_supported(dd); + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) + aspm_ctx_init(rcd); + hfi1_rcd_put(rcd); + } + + /* Start with ASPM disabled */ + aspm_hw_set_l1_ent_latency(dd); + dd->aspm_enabled = false; + aspm_hw_disable_l1(dd); + + /* Now turn on ASPM if configured */ + aspm_enable_all(dd); +} + +void aspm_exit(struct hfi1_devdata *dd) +{ + aspm_disable_all(dd); + + /* Turn on ASPM on exit to conserve power */ + aspm_enable(dd); +} + diff --git a/drivers/infiniband/hw/hfi1/aspm.h b/drivers/infiniband/hw/hfi1/aspm.h index e8133870ee87..75d5d18da3da 100644 --- a/drivers/infiniband/hw/hfi1/aspm.h +++ b/drivers/infiniband/hw/hfi1/aspm.h @@ -57,266 +57,20 @@ enum aspm_mode { ASPM_MODE_DYNAMIC = 2, /* ASPM enabled/disabled dynamically */ }; -/* Time after which the timer interrupt will re-enable ASPM */ -#define ASPM_TIMER_MS 1000 -/* Time for which interrupts are ignored after a timer has been scheduled */ -#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) -/* Two interrupts within this time trigger ASPM disable */ -#define ASPM_TRIGGER_MS 1 -#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) -#define ASPM_L1_SUPPORTED(reg) \ - (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) +void aspm_init(struct hfi1_devdata *dd); +void aspm_exit(struct hfi1_devdata *dd); +void aspm_hw_disable_l1(struct hfi1_devdata *dd); +void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd); +void aspm_disable_all(struct hfi1_devdata *dd); +void aspm_enable_all(struct hfi1_devdata *dd); -static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - u32 up, dn; - - /* - * If the driver does not have access to the upstream component, - * it cannot support ASPM L1 at all. - */ - if (!parent) - return false; - - pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); - dn = ASPM_L1_SUPPORTED(dn); - - pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); - up = ASPM_L1_SUPPORTED(up); - - /* ASPM works on A-step but is reported as not supported */ - return (!!dn || is_ax(dd)) && !!up; -} - -/* Set L1 entrance latency for slower entry to L1 */ -static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) -{ - u32 l1_ent_lat = 0x4u; - u32 reg32; - - pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); - reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; - reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; - pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); -} - -static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - - /* - * If the driver does not have access to the upstream component, - * it cannot support ASPM L1 at all. - */ - if (!parent) - return; - - /* Enable ASPM L1 first in upstream component and then downstream */ - pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, - PCI_EXP_LNKCTL_ASPM_L1); - pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, - PCI_EXP_LNKCTL_ASPM_L1); -} - -static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - - /* Disable ASPM L1 first in downstream component and then upstream */ - pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0x0); - if (parent) - pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0x0); -} - -static inline void aspm_enable(struct hfi1_devdata *dd) -{ - if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || - !dd->aspm_supported) - return; - - aspm_hw_enable_l1(dd); - dd->aspm_enabled = true; -} - -static inline void aspm_disable(struct hfi1_devdata *dd) -{ - if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) - return; - - aspm_hw_disable_l1(dd); - dd->aspm_enabled = false; -} - -static inline void aspm_disable_inc(struct hfi1_devdata *dd) -{ - unsigned long flags; - - spin_lock_irqsave(&dd->aspm_lock, flags); - aspm_disable(dd); - atomic_inc(&dd->aspm_disabled_cnt); - spin_unlock_irqrestore(&dd->aspm_lock, flags); -} - -static inline void aspm_enable_dec(struct hfi1_devdata *dd) -{ - unsigned long flags; - - spin_lock_irqsave(&dd->aspm_lock, flags); - if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) - aspm_enable(dd); - spin_unlock_irqrestore(&dd->aspm_lock, flags); -} - -/* ASPM processing for each receive context interrupt */ static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd) { - bool restart_timer; - bool close_interrupts; - unsigned long flags; - ktime_t now, prev; - /* Quickest exit for minimum impact */ - if (!rcd->aspm_intr_supported) - return; - - spin_lock_irqsave(&rcd->aspm_lock, flags); - /* PSM contexts are open */ - if (!rcd->aspm_intr_enable) - goto unlock; - - prev = rcd->aspm_ts_last_intr; - now = ktime_get(); - rcd->aspm_ts_last_intr = now; - - /* An interrupt pair close together in time */ - close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; - - /* Don't push out our timer till this much time has elapsed */ - restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > - ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; - restart_timer = restart_timer && close_interrupts; - - /* Disable ASPM and schedule timer */ - if (rcd->aspm_enabled && close_interrupts) { - aspm_disable_inc(rcd->dd); - rcd->aspm_enabled = false; - restart_timer = true; - } - - if (restart_timer) { - mod_timer(&rcd->aspm_timer, - jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); - rcd->aspm_ts_timer_sched = now; - } -unlock: - spin_unlock_irqrestore(&rcd->aspm_lock, flags); -} - -/* Timer function for re-enabling ASPM in the absence of interrupt activity */ -static inline void aspm_ctx_timer_function(struct timer_list *t) -{ - struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); - unsigned long flags; - - spin_lock_irqsave(&rcd->aspm_lock, flags); - aspm_enable_dec(rcd->dd); - rcd->aspm_enabled = true; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); -} - -/* - * Disable interrupt processing for verbs contexts when PSM or VNIC contexts - * are open. - */ -static inline void aspm_disable_all(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - unsigned long flags; - u16 i; - - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) { - del_timer_sync(&rcd->aspm_timer); - spin_lock_irqsave(&rcd->aspm_lock, flags); - rcd->aspm_intr_enable = false; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); - hfi1_rcd_put(rcd); - } - } - - aspm_disable(dd); - atomic_set(&dd->aspm_disabled_cnt, 0); -} - -/* Re-enable interrupt processing for verbs contexts */ -static inline void aspm_enable_all(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - unsigned long flags; - u16 i; - - aspm_enable(dd); - - if (aspm_mode != ASPM_MODE_DYNAMIC) + if (likely(!rcd->aspm_intr_supported)) return; - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) { - spin_lock_irqsave(&rcd->aspm_lock, flags); - rcd->aspm_intr_enable = true; - rcd->aspm_enabled = true; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); - hfi1_rcd_put(rcd); - } - } -} - -static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd) -{ - spin_lock_init(&rcd->aspm_lock); - timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); - rcd->aspm_intr_supported = rcd->dd->aspm_supported && - aspm_mode == ASPM_MODE_DYNAMIC && - rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; -} - -static inline void aspm_init(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - u16 i; - - spin_lock_init(&dd->aspm_lock); - dd->aspm_supported = aspm_hw_l1_supported(dd); - - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) - aspm_ctx_init(rcd); - hfi1_rcd_put(rcd); - } - - /* Start with ASPM disabled */ - aspm_hw_set_l1_ent_latency(dd); - dd->aspm_enabled = false; - aspm_hw_disable_l1(dd); - - /* Now turn on ASPM if configured */ - aspm_enable_all(dd); -} - -static inline void aspm_exit(struct hfi1_devdata *dd) -{ - aspm_disable_all(dd); - - /* Turn on ASPM on exit to conserve power */ - aspm_enable(dd); + __aspm_ctx_disable(rcd); } #endif /* _ASPM_H */ diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index c96d193bb236..61aa5504d7c3 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -450,10 +450,6 @@ static int hfi1_pcie_caps; module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444); MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)"); -uint aspm_mode = ASPM_MODE_DISABLED; -module_param_named(aspm, aspm_mode, uint, 0444); -MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); - /** * tune_pcie_caps() - Code to adjust PCIe capabilities. * @dd: Valid device data structure -- cgit v1.2.3 From aa9b79ec37789d7cc1ca1339369b75dc5fec02dd Mon Sep 17 00:00:00 2001 From: Mike Marciniszyn Date: Fri, 28 Jun 2019 14:22:23 -0400 Subject: IB/hfi1: Add missing INVALIDATE opcodes for trace This was missed in the original implementation of the memory management extensions. Fixes: 0db3dfa03c08 ("IB/hfi1: Work request processing for fast register mr and invalidate") Reviewed-by: Michael J. Ruhl Signed-off-by: Mike Marciniszyn Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/trace_ibhdrs.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h index d1372cc66de6..2f84290a88ca 100644 --- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h +++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h @@ -79,6 +79,8 @@ __print_symbolic(opcode, \ ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \ ib_opcode_name(RC_COMPARE_SWAP), \ ib_opcode_name(RC_FETCH_ADD), \ + ib_opcode_name(RC_SEND_LAST_WITH_INVALIDATE), \ + ib_opcode_name(RC_SEND_ONLY_WITH_INVALIDATE), \ ib_opcode_name(TID_RDMA_WRITE_REQ), \ ib_opcode_name(TID_RDMA_WRITE_RESP), \ ib_opcode_name(TID_RDMA_WRITE_DATA), \ -- cgit v1.2.3 From 09fbca8e6240e945c663af1ac2c5d5ef1456bad7 Mon Sep 17 00:00:00 2001 From: Dennis Dalessandro Date: Fri, 28 Jun 2019 14:22:46 -0400 Subject: IB/hfi1: No need to use try_module_get for debugfs The call in debugfs.c for try_module_get() is not needed. A reference to the module will be taken by the VFS layer as long as the owner field is set in the file ops struct. So set this as well as remove the call. Suggested-by: Jason Gunthorpe Reviewed-by: Mike Marciniszyn Signed-off-by: Dennis Dalessandro Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hfi1/debugfs.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c index 15efb4a380b2..d268bf9c42ee 100644 --- a/drivers/infiniband/hw/hfi1/debugfs.c +++ b/drivers/infiniband/hw/hfi1/debugfs.c @@ -987,9 +987,6 @@ static int __i2c_debugfs_open(struct inode *in, struct file *fp, u32 target) struct hfi1_pportdata *ppd; int ret; - if (!try_module_get(THIS_MODULE)) - return -ENODEV; - ppd = private2ppd(fp); ret = acquire_chip_resource(ppd->dd, i2c_target(target), 0); @@ -1155,6 +1152,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp) { \ .name = nm, \ .ops = { \ + .owner = THIS_MODULE, \ .read = readroutine, \ .write = writeroutine, \ .llseek = generic_file_llseek, \ @@ -1165,6 +1163,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp) { \ .name = nm, \ .ops = { \ + .owner = THIS_MODULE, \ .read = readf, \ .write = writef, \ .llseek = generic_file_llseek, \ -- cgit v1.2.3 From 4c44d4634b5c90993fccca9f155347221df6f877 Mon Sep 17 00:00:00 2001 From: Fuqian Huang Date: Fri, 28 Jun 2019 10:47:19 +0800 Subject: IB: Remove unneeded memset In commit af7ddd8a627c ("Merge tag 'dma-mapping-4.21' of git://git.infradead.org/users/hch/dma-mapping"), dma_alloc_coherent/dmam_alloc_coherent always zeroed the returned memory. So the memset after a coherent allocation function is not needed. Signed-off-by: Fuqian Huang Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/cxgb3/cxio_hal.c | 3 --- drivers/infiniband/hw/cxgb4/cq.c | 1 - drivers/infiniband/hw/cxgb4/qp.c | 1 - drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 1 - drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 3 --- drivers/infiniband/hw/mthca/mthca_allocator.c | 2 -- drivers/infiniband/hw/ocrdma/ocrdma_hw.c | 3 --- 7 files changed, 14 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 37ee93824349..95b22a651673 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -174,7 +174,6 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel) return -ENOMEM; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); - memset(cq->queue, 0, size); setup.id = cq->cqid; setup.base_addr = (u64) (cq->dma_addr); setup.size = 1UL << cq->size_log2; @@ -522,8 +521,6 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) dma_unmap_addr_set(&rdev_p->ctrl_qp, mapping, rdev_p->ctrl_qp.dma_addr); rdev_p->ctrl_qp.doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr; - memset(rdev_p->ctrl_qp.workq, 0, - (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr)); mutex_init(&rdev_p->ctrl_qp.lock); init_waitqueue_head(&rdev_p->ctrl_qp.waitq); diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 3cc4d3331a3f..b1bb61c65f4f 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -102,7 +102,6 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, goto err3; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); - memset(cq->queue, 0, cq->memsize); if (user && ucontext->is_32b_cqe) { cq->qp_errp = &((struct t4_status_page *) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index 9523986d62b1..eb9368be28c1 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -274,7 +274,6 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, (unsigned long long)virt_to_phys(wq->sq.queue), wq->rq.queue, (unsigned long long)virt_to_phys(wq->rq.queue)); - memset(wq->rq.queue, 0, wq->rq.memsize); dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); } diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 056a6873df7a..998431c39b8d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -4265,7 +4265,6 @@ static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, } eq->buf_list[i].map = tmp_dma_addr; - memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE); } eq->cons_index = 0; roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 7f2da5e10e67..107330df6ce8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1818,7 +1818,6 @@ static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, goto err_alloc_buf_failed; link_tbl->pg_list[i].map = t; - memset(link_tbl->pg_list[i].buf, 0, buf_chk_sz); entry[i].blk_ba0 = (t >> 12) & 0xffffffff; roce_set_field(entry[i].blk_ba1_nxt_ptr, @@ -5467,8 +5466,6 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, eq->cur_eqe_ba = eq->l0_dma; eq->nxt_eqe_ba = 0; - memset(eq->bt_l0, 0, eq->entries * eq->eqe_size); - return 0; } diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c index aaf10dd5364d..aef1d274a14e 100644 --- a/drivers/infiniband/hw/mthca/mthca_allocator.c +++ b/drivers/infiniband/hw/mthca/mthca_allocator.c @@ -214,8 +214,6 @@ int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, dma_unmap_addr_set(&buf->direct, mapping, t); - memset(buf->direct.buf, 0, size); - while (t & ((1 << shift) - 1)) { --shift; npages *= 2; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index b2dd4e0a4be2..d82d3ec3649e 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1351,7 +1351,6 @@ static int ocrdma_mbx_get_ctrl_attribs(struct ocrdma_dev *dev) mqe->u.nonemb_req.sge[0].pa_hi = (u32) upper_32_bits(dma.pa); mqe->u.nonemb_req.sge[0].len = dma.size; - memset(dma.va, 0, dma.size); ocrdma_init_mch((struct ocrdma_mbx_hdr *)dma.va, OCRDMA_CMD_GET_CTRL_ATTRIBUTES, OCRDMA_SUBSYS_COMMON, @@ -1690,7 +1689,6 @@ static int ocrdma_mbx_create_ah_tbl(struct ocrdma_dev *dev) goto mem_err_ah; dev->av_tbl.pa = pa; dev->av_tbl.num_ah = max_ah; - memset(dev->av_tbl.va, 0, dev->av_tbl.size); pbes = (struct ocrdma_pbe *)dev->av_tbl.pbl.va; for (i = 0; i < dev->av_tbl.size / OCRDMA_MIN_Q_PAGE_SIZE; i++) { @@ -2903,7 +2901,6 @@ static int ocrdma_mbx_get_dcbx_config(struct ocrdma_dev *dev, u32 ptype, mqe_sge->pa_hi = (u32) upper_32_bits(pa); mqe_sge->len = cmd.hdr.pyld_len; - memset(req, 0, sizeof(struct ocrdma_get_dcbx_cfg_req)); ocrdma_init_mch(&req->hdr, OCRDMA_CMD_GET_DCBX_CONFIG, OCRDMA_SUBSYS_DCBX, cmd.hdr.pyld_len); req->param_type = ptype; -- cgit v1.2.3 From cda8cf56d8e29c70dc8a3d989846c66ed1638e74 Mon Sep 17 00:00:00 2001 From: Fuqian Huang Date: Thu, 4 Jul 2019 00:27:42 +0800 Subject: IB/i40iw: Use kmemdup rather than open coding Use kmemdump instead of kzmalloc + memcpy. Signed-off-by: Fuqian Huang Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/i40iw/i40iw_cm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index 8233f5a4e623..84b3ff2687fb 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -4276,11 +4276,11 @@ static void i40iw_qhash_ctrl(struct i40iw_device *iwdev, /* if not found then add a child listener if interface is going up */ if (!ifup) return; - child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC); + child_listen_node = kmemdup(parent_listen_node, + sizeof(*child_listen_node), GFP_ATOMIC); if (!child_listen_node) return; node_allocated = true; - memcpy(child_listen_node, parent_listen_node, sizeof(*child_listen_node)); memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16); -- cgit v1.2.3 From 2e67e775845373905d2c2aecb9062c2c4352a535 Mon Sep 17 00:00:00 2001 From: "Liu, Changcheng" Date: Fri, 28 Jun 2019 14:16:13 +0800 Subject: RDMA/i40iw: Set queue pair state when being queried The API for ib_query_qp requires the driver to set qp_state and cur_qp_state on return, add the missing sets. Fixes: d37498417947 ("i40iw: add files for iwarp interface") Signed-off-by: Changcheng Liu Acked-by: Shiraz Saleem Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/i40iw/i40iw_verbs.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 3c0c6aabc64e..d169a8031375 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -772,6 +772,8 @@ static int i40iw_query_qp(struct ib_qp *ibqp, struct i40iw_qp *iwqp = to_iwqp(ibqp); struct i40iw_sc_qp *qp = &iwqp->sc_qp; + attr->qp_state = iwqp->ibqp_state; + attr->cur_qp_state = attr->qp_state; attr->qp_access_flags = 0; attr->cap.max_send_wr = qp->qp_uk.sq_size; attr->cap.max_recv_wr = qp->qp_uk.rq_size; -- cgit v1.2.3 From 6044414fa849e14fa0de60a75e3f85ea048c89db Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 3 Jul 2019 03:10:21 +0000 Subject: RDMA/hns: Remove set but not used variable 'fclr_write_fail_flag' Fixes gcc '-Wunused-but-set-variable' warning: drivers/infiniband/hw/hns/hns_roce_hw_v2.c: In function 'hns_roce_function_clear': drivers/infiniband/hw/hns/hns_roce_hw_v2.c:1135:7: warning: variable 'fclr_write_fail_flag' set but not used [-Wunused-but-set-variable] It is never used, so can be removed. Signed-off-by: YueHaibing Reviewed-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 107330df6ce8..20e6b5139ef4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1132,7 +1132,6 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) { - bool fclr_write_fail_flag = false; struct hns_roce_func_clear *resp; struct hns_roce_cmq_desc desc; unsigned long end; @@ -1143,7 +1142,6 @@ static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) ret = hns_roce_cmq_send(hr_dev, &desc, 1); if (ret) { - fclr_write_fail_flag = true; dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", ret); return; -- cgit v1.2.3 From 2f40cf30c8644360d37287861d5288f00eab35e5 Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Sun, 30 Jun 2019 10:52:52 +0300 Subject: IB/mlx5: Fixed reporting counters on 2nd port for Dual port RoCE Currently during dual port IB device registration in below code flow, ib_register_device() ib_device_register_sysfs() ib_setup_port_attrs() add_port() get_counter_table() get_perf_mad() process_mad() mlx5_ib_process_mad() mlx5_ib_process_mad() fails on 2nd port when both the ports are not fully setup at the device level (because 2nd port is unaffiliated). As a result, get_perf_mad() registers different PMA counter group for 1st and 2nd port, namely pma_counter_ext and pma_counter. However both ports have the same capability and counter offsets. Due to this when counters are read by the user via sysfs in below code flow, counters are queried from wrong location from the device mainly from PPCNT instead of VPORT counters. show_pma_counter() get_perf_mad() process_mad() mlx5_ib_process_mad() process_pma_cmd() This shows all zero counters for 2nd port. To overcome this, process_pma_cmd() is invoked, and when unaffiliated port is not yet setup during device registration phase, make the query on the first port. while at it, only process_pma_cmd() needs to work on the native port number and underlying mdev, so shift the get, put calls to where its needed inside process_pma_cmd(). Fixes: 212f2a87b74f ("IB/mlx5: Route MADs for dual port RoCE") Signed-off-by: Parav Pandit Reviewed-by: Daniel Jurgens Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mad.c | 60 ++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 24 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index 6c529e6f3a01..348c1df69cdc 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -200,19 +200,33 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt, vl_15_dropped); } -static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, +static int process_pma_cmd(struct mlx5_ib_dev *dev, u8 port_num, const struct ib_mad *in_mad, struct ib_mad *out_mad) { - int err; + struct mlx5_core_dev *mdev; + bool native_port = true; + u8 mdev_port_num; void *out_cnt; + int err; + mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num); + if (!mdev) { + /* Fail to get the native port, likely due to 2nd port is still + * unaffiliated. In such case default to 1st port and attached + * PF device. + */ + native_port = false; + mdev = dev->mdev; + mdev_port_num = 1; + } /* Declaring support of extended counters */ if (in_mad->mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) { struct ib_class_port_info cpi = {}; cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH; memcpy((out_mad->data + 40), &cpi, sizeof(cpi)); - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + goto done; } if (in_mad->mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT) { @@ -221,11 +235,13 @@ static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, int sz = MLX5_ST_SZ_BYTES(query_vport_counter_out); out_cnt = kvzalloc(sz, GFP_KERNEL); - if (!out_cnt) - return IB_MAD_RESULT_FAILURE; + if (!out_cnt) { + err = IB_MAD_RESULT_FAILURE; + goto done; + } err = mlx5_core_query_vport_counter(mdev, 0, 0, - port_num, out_cnt, sz); + mdev_port_num, out_cnt, sz); if (!err) pma_cnt_ext_assign(pma_cnt_ext, out_cnt); } else { @@ -234,20 +250,23 @@ static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); out_cnt = kvzalloc(sz, GFP_KERNEL); - if (!out_cnt) - return IB_MAD_RESULT_FAILURE; + if (!out_cnt) { + err = IB_MAD_RESULT_FAILURE; + goto done; + } - err = mlx5_core_query_ib_ppcnt(mdev, port_num, + err = mlx5_core_query_ib_ppcnt(mdev, mdev_port_num, out_cnt, sz); if (!err) pma_cnt_assign(pma_cnt, out_cnt); - } - + } kvfree(out_cnt); - if (err) - return IB_MAD_RESULT_FAILURE; - - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + err = err ? IB_MAD_RESULT_FAILURE : + IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +done: + if (native_port) + mlx5_ib_put_native_port_mdev(dev, port_num); + return err; } int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, @@ -259,8 +278,6 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, struct mlx5_ib_dev *dev = to_mdev(ibdev); const struct ib_mad *in_mad = (const struct ib_mad *)in; struct ib_mad *out_mad = (struct ib_mad *)out; - struct mlx5_core_dev *mdev; - u8 mdev_port_num; int ret; if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) || @@ -269,19 +286,14 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, memset(out_mad->data, 0, sizeof(out_mad->data)); - mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num); - if (!mdev) - return IB_MAD_RESULT_FAILURE; - - if (MLX5_CAP_GEN(mdev, vport_counters) && + if (MLX5_CAP_GEN(dev->mdev, vport_counters) && in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT && in_mad->mad_hdr.method == IB_MGMT_METHOD_GET) { - ret = process_pma_cmd(mdev, mdev_port_num, in_mad, out_mad); + ret = process_pma_cmd(dev, port_num, in_mad, out_mad); } else { ret = process_mad(ibdev, mad_flags, port_num, in_wc, in_grh, in_mad, out_mad); } - mlx5_ib_put_native_port_mdev(dev, port_num); return ret; } -- cgit v1.2.3 From 2afc5e1b9c340ff20848c8dd8fb60342617bce52 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:29 +0300 Subject: IB/mlx5: Introduce MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD Introduce MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD and its initial implementation. This object is from type class FD and will be used to read DEVX async events. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 95 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 931f587dfb8f..ed01523f0f02 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -33,6 +33,17 @@ struct devx_async_data { struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; }; +struct devx_async_event_file { + struct ib_uobject uobj; + /* Head of events that are subscribed to this FD */ + struct list_head subscribed_events_list; + spinlock_t lock; + wait_queue_head_t poll_wait; + struct list_head event_list; + struct mlx5_ib_dev *dev; + u8 omit_data:1; +}; + #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) struct devx_obj { struct mlx5_core_dev *mdev; @@ -1365,6 +1376,37 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)( return 0; } +static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject( + attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE); + struct devx_async_event_file *ev_file; + struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( + &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); + struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); + u32 flags; + int err; + + err = uverbs_get_flags32(&flags, attrs, + MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS, + MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA); + + if (err) + return err; + + ev_file = container_of(uobj, struct devx_async_event_file, + uobj); + spin_lock_init(&ev_file->lock); + INIT_LIST_HEAD(&ev_file->event_list); + init_waitqueue_head(&ev_file->poll_wait); + if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA) + ev_file->omit_data = 1; + INIT_LIST_HEAD(&ev_file->subscribed_events_list); + ev_file->dev = dev; + return 0; +} + static void devx_query_callback(int status, struct mlx5_async_work *context) { struct devx_async_data *async_data = @@ -1719,6 +1761,32 @@ static const struct file_operations devx_async_cmd_event_fops = { .llseek = no_llseek, }; +static ssize_t devx_async_event_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + return -EINVAL; +} + +static __poll_t devx_async_event_poll(struct file *filp, + struct poll_table_struct *wait) +{ + return 0; +} + +static int devx_async_event_close(struct inode *inode, struct file *filp) +{ + uverbs_close_fd(filp); + return 0; +} + +static const struct file_operations devx_async_event_fops = { + .owner = THIS_MODULE, + .read = devx_async_event_read, + .poll = devx_async_event_poll, + .release = devx_async_event_close, + .llseek = no_llseek, +}; + static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, enum rdma_remove_reason why) { @@ -1738,6 +1806,12 @@ static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, return 0; }; +static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + return 0; +}; + DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_DEVX_UMEM_REG, UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE, @@ -1903,6 +1977,24 @@ DECLARE_UVERBS_NAMED_OBJECT( O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC, + UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE, + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_ACCESS_NEW, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS, + enum mlx5_ib_uapi_devx_create_event_channel_flags, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT( + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file), + devx_hot_unplug_async_event_file, + &devx_async_event_fops, "[devx_async_event]", + O_RDONLY), + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)); + static bool devx_is_supported(struct ib_device *device) { struct mlx5_ib_dev *dev = to_mdev(device); @@ -1923,5 +2015,8 @@ const struct uapi_definition mlx5_ib_devx_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED( MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), + UAPI_DEF_CHAIN_OBJ_TREE_NAMED( + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), {}, }; -- cgit v1.2.3 From e337dd53ce4cc3db79e52704e554f648c46d5e91 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:30 +0300 Subject: IB/mlx5: Register DEVX with mlx5_core to get async events Register DEVX with with mlx5_core to get async events. This will enable to dispatch the applicable events to its consumers in down stream patches. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 30 ++++++++++++++++++++++++++++++ drivers/infiniband/hw/mlx5/main.c | 8 ++++++-- drivers/infiniband/hw/mlx5/mlx5_ib.h | 12 ++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index ed01523f0f02..a9affc905bfa 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -1663,6 +1663,36 @@ static int devx_umem_cleanup(struct ib_uobject *uobject, return 0; } +static int devx_event_notifier(struct notifier_block *nb, + unsigned long event_type, void *data) +{ + return NOTIFY_DONE; +} + +void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) +{ + struct mlx5_devx_event_table *table = &dev->devx_event_table; + + xa_init(&table->event_xa); + mutex_init(&table->event_xa_lock); + MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY); + mlx5_eq_notifier_register(dev->mdev, &table->devx_nb); +} + +void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) +{ + struct mlx5_devx_event_table *table = &dev->devx_event_table; + void *entry; + unsigned long id; + + mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb); + + xa_for_each(&table->event_xa, id, entry) + kfree(entry); + + xa_destroy(&table->event_xa); +} + static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 9db8c06aa01e..692b60898ee2 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6630,15 +6630,19 @@ static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev) int uid; uid = mlx5_ib_devx_create(dev, false); - if (uid > 0) + if (uid > 0) { dev->devx_whitelist_uid = uid; + mlx5_ib_devx_init_event_table(dev); + } return 0; } static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev) { - if (dev->devx_whitelist_uid) + if (dev->devx_whitelist_uid) { + mlx5_ib_devx_cleanup_event_table(dev); mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid); + } } void __mlx5_ib_remove(struct mlx5_ib_dev *dev, diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 305d26cdf7f3..7373e9da0919 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -936,6 +936,13 @@ struct mlx5_ib_pf_eq { mempool_t *pool; }; +struct mlx5_devx_event_table { + struct mlx5_nb devx_nb; + /* serialize updating the event_xa */ + struct mutex event_xa_lock; + struct xarray event_xa; +}; + struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; @@ -985,6 +992,7 @@ struct mlx5_ib_dev { u16 devx_whitelist_uid; struct mlx5_srq_table srq_table; struct mlx5_async_ctx async_ctx; + struct mlx5_devx_event_table devx_event_table; }; static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq) @@ -1324,6 +1332,8 @@ void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev, #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); +void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); +void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void); extern const struct uapi_definition mlx5_ib_devx_defs[]; extern const struct uapi_definition mlx5_ib_flow_defs[]; @@ -1341,6 +1351,8 @@ static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) { return -EOPNOTSUPP; } static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {} +static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {} +static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {} static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) { -- cgit v1.2.3 From 7597385371425febdaa8c6a1da3625d4ffff16f5 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:31 +0300 Subject: IB/mlx5: Enable subscription for device events over DEVX Enable subscription for device events over DEVX. Each subscription is added to the two level xarray data structure according to its event number and the DEVX object information in case was given with the given target fd. Those events will be reported over the given fd once will occur. Downstream patches will mange the dispatching to any subscription. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 560 +++++++++++++++++++++++++++++++++++++- 1 file changed, 553 insertions(+), 7 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index a9affc905bfa..9c21cafc44a6 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -14,6 +14,7 @@ #include #include #include "mlx5_ib.h" +#include #define UVERBS_MODULE_NAME mlx5_ib #include @@ -33,6 +34,40 @@ struct devx_async_data { struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; }; +/* first level XA value data structure */ +struct devx_event { + struct xarray object_ids; /* second XA level, Key = object id */ + struct list_head unaffiliated_list; +}; + +/* second level XA value data structure */ +struct devx_obj_event { + struct rcu_head rcu; + struct list_head obj_sub_list; +}; + +struct devx_event_subscription { + struct list_head file_list; /* headed in ev_file-> + * subscribed_events_list + */ + struct list_head xa_list; /* headed in devx_event->unaffiliated_list or + * devx_obj_event->obj_sub_list + */ + struct list_head obj_list; /* headed in devx_object */ + struct list_head event_list; /* headed in ev_file->event_list or in + * temp list via subscription + */ + + u8 is_cleaned:1; + u32 xa_key_level1; + u32 xa_key_level2; + struct rcu_head rcu; + u64 cookie; + struct devx_async_event_file *ev_file; + struct file *filp; /* Upon hot unplug we need a direct access to */ + struct eventfd_ctx *eventfd; +}; + struct devx_async_event_file { struct ib_uobject uobj; /* Head of events that are subscribed to this FD */ @@ -55,6 +90,7 @@ struct devx_obj { struct mlx5_ib_devx_mr devx_mr; struct mlx5_core_dct core_dct; }; + struct list_head event_sub; /* holds devx_event_subscription entries */ }; struct devx_umem { @@ -160,6 +196,104 @@ bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id) return false; } +static bool is_legacy_unaffiliated_event_num(u16 event_num) +{ + switch (event_num) { + case MLX5_EVENT_TYPE_PORT_CHANGE: + return true; + default: + return false; + } +} + +static bool is_legacy_obj_event_num(u16 event_num) +{ + switch (event_num) { + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_CQ_ERROR: + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_DCT_DRAINED: + case MLX5_EVENT_TYPE_COMP: + return true; + default: + return false; + } +} + +static u16 get_legacy_obj_type(u16 opcode) +{ + switch (opcode) { + case MLX5_CMD_OP_CREATE_RQ: + return MLX5_EVENT_QUEUE_TYPE_RQ; + case MLX5_CMD_OP_CREATE_QP: + return MLX5_EVENT_QUEUE_TYPE_QP; + case MLX5_CMD_OP_CREATE_SQ: + return MLX5_EVENT_QUEUE_TYPE_SQ; + case MLX5_CMD_OP_CREATE_DCT: + return MLX5_EVENT_QUEUE_TYPE_DCT; + default: + return 0; + } +} + +static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num) +{ + u16 opcode; + + opcode = (obj->obj_id >> 32) & 0xffff; + + if (is_legacy_obj_event_num(event_num)) + return get_legacy_obj_type(opcode); + + switch (opcode) { + case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: + return (obj->obj_id >> 48); + case MLX5_CMD_OP_CREATE_RQ: + return MLX5_OBJ_TYPE_RQ; + case MLX5_CMD_OP_CREATE_QP: + return MLX5_OBJ_TYPE_QP; + case MLX5_CMD_OP_CREATE_SQ: + return MLX5_OBJ_TYPE_SQ; + case MLX5_CMD_OP_CREATE_DCT: + return MLX5_OBJ_TYPE_DCT; + case MLX5_CMD_OP_CREATE_TIR: + return MLX5_OBJ_TYPE_TIR; + case MLX5_CMD_OP_CREATE_TIS: + return MLX5_OBJ_TYPE_TIS; + case MLX5_CMD_OP_CREATE_PSV: + return MLX5_OBJ_TYPE_PSV; + case MLX5_OBJ_TYPE_MKEY: + return MLX5_OBJ_TYPE_MKEY; + case MLX5_CMD_OP_CREATE_RMP: + return MLX5_OBJ_TYPE_RMP; + case MLX5_CMD_OP_CREATE_XRC_SRQ: + return MLX5_OBJ_TYPE_XRC_SRQ; + case MLX5_CMD_OP_CREATE_XRQ: + return MLX5_OBJ_TYPE_XRQ; + case MLX5_CMD_OP_CREATE_RQT: + return MLX5_OBJ_TYPE_RQT; + case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: + return MLX5_OBJ_TYPE_FLOW_COUNTER; + case MLX5_CMD_OP_CREATE_CQ: + return MLX5_OBJ_TYPE_CQ; + default: + return 0; + } +} + +static u32 get_dec_obj_id(u64 obj_id) +{ + return (obj_id & 0xffffffff); +} + /* * As the obj_id in the firmware is not globally unique the object type * must be considered upon checking for a valid object id. @@ -1126,14 +1260,47 @@ static void devx_cleanup_mkey(struct devx_obj *obj) mlx5_base_mkey(obj->devx_mr.mmkey.key)); } +static void devx_cleanup_subscription(struct mlx5_ib_dev *dev, + struct devx_event_subscription *sub) +{ + struct devx_event *event; + struct devx_obj_event *xa_val_level2; + + if (sub->is_cleaned) + return; + + sub->is_cleaned = 1; + list_del_rcu(&sub->xa_list); + + if (list_empty(&sub->obj_list)) + return; + + list_del_rcu(&sub->obj_list); + /* check whether key level 1 for this obj_sub_list is empty */ + event = xa_load(&dev->devx_event_table.event_xa, + sub->xa_key_level1); + WARN_ON(!event); + + xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2); + if (list_empty(&xa_val_level2->obj_sub_list)) { + xa_erase(&event->object_ids, + sub->xa_key_level2); + kfree_rcu(xa_val_level2, rcu); + } +} + static int devx_obj_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_devx_event_table *devx_event_table; struct devx_obj *obj = uobject->object; + struct devx_event_subscription *sub_entry, *tmp; + struct mlx5_ib_dev *dev; int ret; + dev = mlx5_udata_to_mdev(&attrs->driver_udata); if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) devx_cleanup_mkey(obj); @@ -1145,10 +1312,14 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, if (ib_is_destroy_retryable(ret, why, uobject)) return ret; - if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { - struct mlx5_ib_dev *dev = - mlx5_udata_to_mdev(&attrs->driver_udata); + devx_event_table = &dev->devx_event_table; + + mutex_lock(&devx_event_table->event_xa_lock); + list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list) + devx_cleanup_subscription(dev, sub_entry); + mutex_unlock(&devx_event_table->event_xa_lock); + if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu, devx_free_indirect_mkey); return ret; @@ -1220,6 +1391,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( uobj->object = obj; obj->mdev = dev->mdev; + INIT_LIST_HEAD(&obj->event_sub); devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, &obj_id); WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); @@ -1404,6 +1576,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)( ev_file->omit_data = 1; INIT_LIST_HEAD(&ev_file->subscribed_events_list); ev_file->dev = dev; + get_device(&dev->ib_dev.dev); return 0; } @@ -1516,6 +1689,331 @@ sub_bytes: return err; } +static void +subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, + u32 key_level1, + bool is_level2, + u32 key_level2) +{ + struct devx_event *event; + struct devx_obj_event *xa_val_level2; + + /* Level 1 is valid for future use, no need to free */ + if (!is_level2) + return; + + event = xa_load(&devx_event_table->event_xa, key_level1); + WARN_ON(!event); + + xa_val_level2 = xa_load(&event->object_ids, + key_level2); + if (list_empty(&xa_val_level2->obj_sub_list)) { + xa_erase(&event->object_ids, + key_level2); + kfree_rcu(xa_val_level2, rcu); + } +} + +static int +subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table, + u32 key_level1, + bool is_level2, + u32 key_level2) +{ + struct devx_obj_event *obj_event; + struct devx_event *event; + int err; + + event = xa_load(&devx_event_table->event_xa, key_level1); + if (!event) { + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return -ENOMEM; + + INIT_LIST_HEAD(&event->unaffiliated_list); + xa_init(&event->object_ids); + + err = xa_insert(&devx_event_table->event_xa, + key_level1, + event, + GFP_KERNEL); + if (err) { + kfree(event); + return err; + } + } + + if (!is_level2) + return 0; + + obj_event = xa_load(&event->object_ids, key_level2); + if (!obj_event) { + obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL); + if (!obj_event) + /* Level1 is valid for future use, no need to free */ + return -ENOMEM; + + err = xa_insert(&event->object_ids, + key_level2, + obj_event, + GFP_KERNEL); + if (err) + return err; + INIT_LIST_HEAD(&obj_event->obj_sub_list); + } + + return 0; +} + +static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list, + struct devx_obj *obj) +{ + int i; + + for (i = 0; i < num_events; i++) { + if (obj) { + if (!is_legacy_obj_event_num(event_type_num_list[i])) + return false; + } else if (!is_legacy_unaffiliated_event_num( + event_type_num_list[i])) { + return false; + } + } + + return true; +} + +#define MAX_SUPP_EVENT_NUM 255 +static bool is_valid_events(struct mlx5_core_dev *dev, + int num_events, u16 *event_type_num_list, + struct devx_obj *obj) +{ + __be64 *aff_events; + __be64 *unaff_events; + int mask_entry; + int mask_bit; + int i; + + if (MLX5_CAP_GEN(dev, event_cap)) { + aff_events = MLX5_CAP_DEV_EVENT(dev, + user_affiliated_events); + unaff_events = MLX5_CAP_DEV_EVENT(dev, + user_unaffiliated_events); + } else { + return is_valid_events_legacy(num_events, event_type_num_list, + obj); + } + + for (i = 0; i < num_events; i++) { + if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM) + return false; + + mask_entry = event_type_num_list[i] / 64; + mask_bit = event_type_num_list[i] % 64; + + if (obj) { + /* CQ completion */ + if (event_type_num_list[i] == 0) + continue; + + if (!(be64_to_cpu(aff_events[mask_entry]) & + (1ull << mask_bit))) + return false; + + continue; + } + + if (!(be64_to_cpu(unaff_events[mask_entry]) & + (1ull << mask_bit))) + return false; + } + + return true; +} + +#define MAX_NUM_EVENTS 16 +static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *devx_uobj = uverbs_attr_get_uobject( + attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE); + struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( + &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); + struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); + struct ib_uobject *fd_uobj; + struct devx_obj *obj = NULL; + struct devx_async_event_file *ev_file; + struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table; + u16 *event_type_num_list; + struct devx_event_subscription *event_sub, *tmp_sub; + struct list_head sub_list; + int redirect_fd; + bool use_eventfd = false; + int num_events; + int num_alloc_xa_entries = 0; + u16 obj_type = 0; + u64 cookie = 0; + u32 obj_id = 0; + int err; + int i; + + if (!c->devx_uid) + return -EINVAL; + + if (!IS_ERR(devx_uobj)) { + obj = (struct devx_obj *)devx_uobj->object; + if (obj) + obj_id = get_dec_obj_id(obj->obj_id); + } + + fd_uobj = uverbs_attr_get_uobject(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE); + if (IS_ERR(fd_uobj)) + return PTR_ERR(fd_uobj); + + ev_file = container_of(fd_uobj, struct devx_async_event_file, + uobj); + + if (uverbs_attr_is_valid(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) { + err = uverbs_copy_from(&redirect_fd, attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM); + if (err) + return err; + + use_eventfd = true; + } + + if (uverbs_attr_is_valid(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) { + if (use_eventfd) + return -EINVAL; + + err = uverbs_copy_from(&cookie, attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE); + if (err) + return err; + } + + num_events = uverbs_attr_ptr_get_array_size( + attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST, + sizeof(u16)); + + if (num_events < 0) + return num_events; + + if (num_events > MAX_NUM_EVENTS) + return -EINVAL; + + event_type_num_list = uverbs_attr_get_alloced_ptr(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST); + + if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj)) + return -EINVAL; + + INIT_LIST_HEAD(&sub_list); + + /* Protect from concurrent subscriptions to same XA entries to allow + * both to succeed + */ + mutex_lock(&devx_event_table->event_xa_lock); + for (i = 0; i < num_events; i++) { + u32 key_level1; + + if (obj) + obj_type = get_dec_obj_type(obj, + event_type_num_list[i]); + key_level1 = event_type_num_list[i] | obj_type << 16; + + err = subscribe_event_xa_alloc(devx_event_table, + key_level1, + obj, + obj_id); + if (err) + goto err; + + num_alloc_xa_entries++; + event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL); + if (!event_sub) + goto err; + + list_add_tail(&event_sub->event_list, &sub_list); + if (use_eventfd) { + event_sub->eventfd = + eventfd_ctx_fdget(redirect_fd); + + if (IS_ERR(event_sub)) { + err = PTR_ERR(event_sub->eventfd); + event_sub->eventfd = NULL; + goto err; + } + } + + event_sub->cookie = cookie; + event_sub->ev_file = ev_file; + event_sub->filp = fd_uobj->object; + /* May be needed upon cleanup the devx object/subscription */ + event_sub->xa_key_level1 = key_level1; + event_sub->xa_key_level2 = obj_id; + INIT_LIST_HEAD(&event_sub->obj_list); + } + + /* Once all the allocations and the XA data insertions were done we + * can go ahead and add all the subscriptions to the relevant lists + * without concern of a failure. + */ + list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { + struct devx_event *event; + struct devx_obj_event *obj_event; + + list_del_init(&event_sub->event_list); + + spin_lock_irq(&ev_file->lock); + list_add_tail_rcu(&event_sub->file_list, + &ev_file->subscribed_events_list); + spin_unlock_irq(&ev_file->lock); + + event = xa_load(&devx_event_table->event_xa, + event_sub->xa_key_level1); + WARN_ON(!event); + + if (!obj) { + list_add_tail_rcu(&event_sub->xa_list, + &event->unaffiliated_list); + continue; + } + + obj_event = xa_load(&event->object_ids, obj_id); + WARN_ON(!obj_event); + list_add_tail_rcu(&event_sub->xa_list, + &obj_event->obj_sub_list); + list_add_tail_rcu(&event_sub->obj_list, + &obj->event_sub); + } + + mutex_unlock(&devx_event_table->event_xa_lock); + return 0; + +err: + list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { + list_del(&event_sub->event_list); + + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); + + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + + kfree(event_sub); + } + + mutex_unlock(&devx_event_table->event_xa_lock); + return err; +} + static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, struct uverbs_attr_bundle *attrs, struct devx_umem *obj) @@ -1682,14 +2180,21 @@ void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) { struct mlx5_devx_event_table *table = &dev->devx_event_table; + struct devx_event_subscription *sub, *tmp; + struct devx_event *event; void *entry; unsigned long id; mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb); - - xa_for_each(&table->event_xa, id, entry) + mutex_lock(&dev->devx_event_table.event_xa_lock); + xa_for_each(&table->event_xa, id, entry) { + event = entry; + list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list, + xa_list) + devx_cleanup_subscription(dev, sub); kfree(entry); - + } + mutex_unlock(&dev->devx_event_table.event_xa_lock); xa_destroy(&table->event_xa); } @@ -1805,7 +2310,26 @@ static __poll_t devx_async_event_poll(struct file *filp, static int devx_async_event_close(struct inode *inode, struct file *filp) { + struct devx_async_event_file *ev_file = filp->private_data; + struct devx_event_subscription *event_sub, *event_sub_tmp; + + mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock); + /* delete the subscriptions which are related to this FD */ + list_for_each_entry_safe(event_sub, event_sub_tmp, + &ev_file->subscribed_events_list, file_list) { + devx_cleanup_subscription(ev_file->dev, event_sub); + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + + list_del_rcu(&event_sub->file_list); + /* subscription may not be used by the read API any more */ + kfree_rcu(event_sub, rcu); + } + + mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock); + uverbs_close_fd(filp); + put_device(&ev_file->dev->ib_dev.dev); return 0; } @@ -1973,10 +2497,32 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_TYPE(u64), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT, + UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE, + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE, + MLX5_IB_OBJECT_DEVX_OBJ, + UVERBS_ACCESS_READ, + UA_OPTIONAL), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST, + UVERBS_ATTR_MIN_SIZE(sizeof(u16)), + UA_MANDATORY, + UA_ALLOC_AND_COPY), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE, + UVERBS_ATTR_TYPE(u64), + UA_OPTIONAL), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL)); + DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX, &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR), - &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN)); + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN), + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)); DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ, UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup), -- cgit v1.2.3 From 5ec9d8ee87c627a2c981d871e41f6e2a942f53fd Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:32 +0300 Subject: IB/mlx5: Implement DEVX dispatching event Implement DEVX dispatching event by looking up for the applicable subscriptions for the reported event and using their target fd to signal/set the event. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 303 +++++++++++++++++++++++++++++++++++++- 1 file changed, 300 insertions(+), 3 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 9c21cafc44a6..867b9778c063 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -34,6 +34,11 @@ struct devx_async_data { struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; }; +struct devx_async_event_data { + struct list_head list; /* headed in ev_file->event_list */ + struct mlx5_ib_uapi_devx_async_event_hdr hdr; +}; + /* first level XA value data structure */ struct devx_event { struct xarray object_ids; /* second XA level, Key = object id */ @@ -77,6 +82,8 @@ struct devx_async_event_file { struct list_head event_list; struct mlx5_ib_dev *dev; u8 omit_data:1; + u8 is_overflow_err:1; + u8 is_destroyed:1; }; #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) @@ -289,6 +296,29 @@ static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num) } } +static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe) +{ + switch (event_type) { + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + return eqe->data.qp_srq.type; + case MLX5_EVENT_TYPE_CQ_ERROR: + return 0; + case MLX5_EVENT_TYPE_DCT_DRAINED: + return MLX5_EVENT_QUEUE_TYPE_DCT; + default: + return MLX5_GET(affiliated_event_header, &eqe->data, obj_type); + } +} + static u32 get_dec_obj_id(u64 obj_id) { return (obj_id & 0xffffffff); @@ -2161,10 +2191,170 @@ static int devx_umem_cleanup(struct ib_uobject *uobject, return 0; } +static bool is_unaffiliated_event(struct mlx5_core_dev *dev, + unsigned long event_type) +{ + __be64 *unaff_events; + int mask_entry; + int mask_bit; + + if (!MLX5_CAP_GEN(dev, event_cap)) + return is_legacy_unaffiliated_event_num(event_type); + + unaff_events = MLX5_CAP_DEV_EVENT(dev, + user_unaffiliated_events); + WARN_ON(event_type > MAX_SUPP_EVENT_NUM); + + mask_entry = event_type / 64; + mask_bit = event_type % 64; + + if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit))) + return false; + + return true; +} + +static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data) +{ + struct mlx5_eqe *eqe = data; + u32 obj_id = 0; + + switch (event_type) { + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; + break; + case MLX5_EVENT_TYPE_DCT_DRAINED: + obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff; + break; + case MLX5_EVENT_TYPE_CQ_ERROR: + obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff; + break; + default: + obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id); + break; + } + + return obj_id; +} + +static int deliver_event(struct devx_event_subscription *event_sub, + const void *data) +{ + struct devx_async_event_file *ev_file; + struct devx_async_event_data *event_data; + unsigned long flags; + + ev_file = event_sub->ev_file; + + if (ev_file->omit_data) { + spin_lock_irqsave(&ev_file->lock, flags); + if (!list_empty(&event_sub->event_list)) { + spin_unlock_irqrestore(&ev_file->lock, flags); + return 0; + } + + list_add_tail(&event_sub->event_list, &ev_file->event_list); + spin_unlock_irqrestore(&ev_file->lock, flags); + wake_up_interruptible(&ev_file->poll_wait); + return 0; + } + + event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe), + GFP_ATOMIC); + if (!event_data) { + spin_lock_irqsave(&ev_file->lock, flags); + ev_file->is_overflow_err = 1; + spin_unlock_irqrestore(&ev_file->lock, flags); + return -ENOMEM; + } + + event_data->hdr.cookie = event_sub->cookie; + memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); + + spin_lock_irqsave(&ev_file->lock, flags); + list_add_tail(&event_data->list, &ev_file->event_list); + spin_unlock_irqrestore(&ev_file->lock, flags); + wake_up_interruptible(&ev_file->poll_wait); + + return 0; +} + +static void dispatch_event_fd(struct list_head *fd_list, + const void *data) +{ + struct devx_event_subscription *item; + + list_for_each_entry_rcu(item, fd_list, xa_list) { + if (!get_file_rcu(item->filp)) + continue; + + if (item->eventfd) { + eventfd_signal(item->eventfd, 1); + fput(item->filp); + continue; + } + + deliver_event(item, data); + fput(item->filp); + } +} + static int devx_event_notifier(struct notifier_block *nb, unsigned long event_type, void *data) { - return NOTIFY_DONE; + struct mlx5_devx_event_table *table; + struct mlx5_ib_dev *dev; + struct devx_event *event; + struct devx_obj_event *obj_event; + u16 obj_type = 0; + bool is_unaffiliated; + u32 obj_id; + + /* Explicit filtering to kernel events which may occur frequently */ + if (event_type == MLX5_EVENT_TYPE_CMD || + event_type == MLX5_EVENT_TYPE_PAGE_REQUEST) + return NOTIFY_OK; + + table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb); + dev = container_of(table, struct mlx5_ib_dev, devx_event_table); + is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type); + + if (!is_unaffiliated) + obj_type = get_event_obj_type(event_type, data); + + rcu_read_lock(); + event = xa_load(&table->event_xa, event_type | (obj_type << 16)); + if (!event) { + rcu_read_unlock(); + return NOTIFY_DONE; + } + + if (is_unaffiliated) { + dispatch_event_fd(&event->unaffiliated_list, data); + rcu_read_unlock(); + return NOTIFY_OK; + } + + obj_id = devx_get_obj_id_from_event(event_type, data); + obj_event = xa_load(&event->object_ids, obj_id); + if (!obj_event) { + rcu_read_unlock(); + return NOTIFY_DONE; + } + + dispatch_event_fd(&obj_event->obj_sub_list, data); + + rcu_read_unlock(); + return NOTIFY_OK; } void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) @@ -2299,19 +2489,108 @@ static const struct file_operations devx_async_cmd_event_fops = { static ssize_t devx_async_event_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { - return -EINVAL; + struct devx_async_event_file *ev_file = filp->private_data; + struct devx_event_subscription *event_sub; + struct devx_async_event_data *uninitialized_var(event); + int ret = 0; + size_t eventsz; + bool omit_data; + void *event_data; + + omit_data = ev_file->omit_data; + + spin_lock_irq(&ev_file->lock); + + if (ev_file->is_overflow_err) { + ev_file->is_overflow_err = 0; + spin_unlock_irq(&ev_file->lock); + return -EOVERFLOW; + } + + if (ev_file->is_destroyed) { + spin_unlock_irq(&ev_file->lock); + return -EIO; + } + + while (list_empty(&ev_file->event_list)) { + spin_unlock_irq(&ev_file->lock); + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(ev_file->poll_wait, + (!list_empty(&ev_file->event_list) || + ev_file->is_destroyed))) { + return -ERESTARTSYS; + } + + spin_lock_irq(&ev_file->lock); + if (ev_file->is_destroyed) { + spin_unlock_irq(&ev_file->lock); + return -EIO; + } + } + + if (omit_data) { + event_sub = list_first_entry(&ev_file->event_list, + struct devx_event_subscription, + event_list); + eventsz = sizeof(event_sub->cookie); + event_data = &event_sub->cookie; + } else { + event = list_first_entry(&ev_file->event_list, + struct devx_async_event_data, list); + eventsz = sizeof(struct mlx5_eqe) + + sizeof(struct mlx5_ib_uapi_devx_async_event_hdr); + event_data = &event->hdr; + } + + if (eventsz > count) { + spin_unlock_irq(&ev_file->lock); + return -EINVAL; + } + + if (omit_data) + list_del_init(&event_sub->event_list); + else + list_del(&event->list); + + spin_unlock_irq(&ev_file->lock); + + if (copy_to_user(buf, event_data, eventsz)) + /* This points to an application issue, not a kernel concern */ + ret = -EFAULT; + else + ret = eventsz; + + if (!omit_data) + kfree(event); + return ret; } static __poll_t devx_async_event_poll(struct file *filp, struct poll_table_struct *wait) { - return 0; + struct devx_async_event_file *ev_file = filp->private_data; + __poll_t pollflags = 0; + + poll_wait(filp, &ev_file->poll_wait, wait); + + spin_lock_irq(&ev_file->lock); + if (ev_file->is_destroyed) + pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; + else if (!list_empty(&ev_file->event_list)) + pollflags = EPOLLIN | EPOLLRDNORM; + spin_unlock_irq(&ev_file->lock); + + return pollflags; } static int devx_async_event_close(struct inode *inode, struct file *filp) { struct devx_async_event_file *ev_file = filp->private_data; struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock); /* delete the subscriptions which are related to this FD */ @@ -2328,6 +2607,15 @@ static int devx_async_event_close(struct inode *inode, struct file *filp) mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock); + /* free the pending events allocation */ + if (!ev_file->omit_data) { + spin_lock_irq(&ev_file->lock); + list_for_each_entry_safe(entry, tmp, + &ev_file->event_list, list) + kfree(entry); /* read can't come any more */ + spin_unlock_irq(&ev_file->lock); + } + uverbs_close_fd(filp); put_device(&ev_file->dev->ib_dev.dev); return 0; @@ -2363,6 +2651,15 @@ static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj, enum rdma_remove_reason why) { + struct devx_async_event_file *ev_file = + container_of(uobj, struct devx_async_event_file, + uobj); + + spin_lock_irq(&ev_file->lock); + ev_file->is_destroyed = 1; + spin_unlock_irq(&ev_file->lock); + + wake_up_interruptible(&ev_file->poll_wait); return 0; }; -- cgit v1.2.3 From ef1659ade3590e4a29a999c6f0cb2272857638a6 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:33 +0300 Subject: IB/mlx5: Add DEVX support for CQ events Add DEVX support for CQ events by creating and destroying the CQ via mlx5_core and set an handler to manage its completions. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 867b9778c063..b6cae4ea7a37 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -19,9 +19,12 @@ #define UVERBS_MODULE_NAME mlx5_ib #include +static void dispatch_event_fd(struct list_head *fd_list, const void *data); + enum devx_obj_flags { DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0, DEVX_OBJ_FLAGS_DCT = 1 << 1, + DEVX_OBJ_FLAGS_CQ = 1 << 2, }; struct devx_async_data { @@ -89,6 +92,7 @@ struct devx_async_event_file { #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) struct devx_obj { struct mlx5_core_dev *mdev; + struct mlx5_ib_dev *ib_dev; u64 obj_id; u32 dinlen; /* destroy inbox length */ u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; @@ -96,6 +100,7 @@ struct devx_obj { union { struct mlx5_ib_devx_mr devx_mr; struct mlx5_core_dct core_dct; + struct mlx5_core_cq core_cq; }; struct list_head event_sub; /* holds devx_event_subscription entries */ }; @@ -1336,6 +1341,8 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, if (obj->flags & DEVX_OBJ_FLAGS_DCT) ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + else if (obj->flags & DEVX_OBJ_FLAGS_CQ) + ret = mlx5_core_destroy_cq(obj->mdev, &obj->core_cq); else ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); @@ -1359,6 +1366,29 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, return ret; } +static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) +{ + struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq); + struct mlx5_devx_event_table *table; + struct devx_event *event; + struct devx_obj_event *obj_event; + u32 obj_id = mcq->cqn; + + table = &obj->ib_dev->devx_event_table; + rcu_read_lock(); + event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP); + if (!event) + goto out; + + obj_event = xa_load(&event->object_ids, obj_id); + if (!obj_event) + goto out; + + dispatch_event_fd(&obj_event->obj_sub_list, eqe); +out: + rcu_read_unlock(); +} + static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( struct uverbs_attr_bundle *attrs) { @@ -1410,6 +1440,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( err = mlx5_core_create_dct(dev->mdev, &obj->core_dct, cmd_in, cmd_in_len, cmd_out, cmd_out_len); + } else if (opcode == MLX5_CMD_OP_CREATE_CQ) { + obj->flags |= DEVX_OBJ_FLAGS_CQ; + obj->core_cq.comp = devx_cq_comp; + err = mlx5_core_create_cq(dev->mdev, &obj->core_cq, + cmd_in, cmd_in_len, cmd_out, + cmd_out_len); } else { err = mlx5_cmd_exec(dev->mdev, cmd_in, cmd_in_len, @@ -1422,6 +1458,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( uobj->object = obj; obj->mdev = dev->mdev; INIT_LIST_HEAD(&obj->event_sub); + obj->ib_dev = dev; devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, &obj_id); WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); @@ -1449,6 +1486,8 @@ err_copy: obj_destroy: if (obj->flags & DEVX_OBJ_FLAGS_DCT) mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + else if (obj->flags & DEVX_OBJ_FLAGS_CQ) + mlx5_core_destroy_cq(obj->mdev, &obj->core_cq); else mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); -- cgit v1.2.3 From 5832fdd35e61bf4793da0f0480fb873af645a7b4 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Sun, 30 Jun 2019 19:23:34 +0300 Subject: IB/mlx5: DEVX cleanup mdev No need any more to hold mlx5_core_dev on the devx_object, it can be accessed from ib_dev. Signed-off-by: Yishai Hadas Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index b6cae4ea7a37..e69a8693f102 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -91,7 +91,6 @@ struct devx_async_event_file { #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) struct devx_obj { - struct mlx5_core_dev *mdev; struct mlx5_ib_dev *ib_dev; u64 obj_id; u32 dinlen; /* destroy inbox length */ @@ -1291,7 +1290,7 @@ static void devx_free_indirect_mkey(struct rcu_head *rcu) */ static void devx_cleanup_mkey(struct devx_obj *obj) { - xa_erase(&obj->mdev->priv.mkey_table, + xa_erase(&obj->ib_dev->mdev->priv.mkey_table, mlx5_base_mkey(obj->devx_mr.mmkey.key)); } @@ -1340,12 +1339,12 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, devx_cleanup_mkey(obj); if (obj->flags & DEVX_OBJ_FLAGS_DCT) - ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + ret = mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct); else if (obj->flags & DEVX_OBJ_FLAGS_CQ) - ret = mlx5_core_destroy_cq(obj->mdev, &obj->core_cq); + ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq); else - ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, - sizeof(out)); + ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, + obj->dinlen, out, sizeof(out)); if (ib_is_destroy_retryable(ret, why, uobject)) return ret; @@ -1456,7 +1455,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( goto obj_free; uobj->object = obj; - obj->mdev = dev->mdev; INIT_LIST_HEAD(&obj->event_sub); obj->ib_dev = dev; devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, @@ -1485,11 +1483,11 @@ err_copy: devx_cleanup_mkey(obj); obj_destroy: if (obj->flags & DEVX_OBJ_FLAGS_DCT) - mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct); else if (obj->flags & DEVX_OBJ_FLAGS_CQ) - mlx5_core_destroy_cq(obj->mdev, &obj->core_cq); + mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq); else - mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, + mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); obj_free: kfree(obj); -- cgit v1.2.3 From 9a601fc43ee0157a2575fe81beffe653c25edcd6 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Thu, 4 Jul 2019 14:22:58 +0800 Subject: RDMA/hns: Fix building modular hns The patch below wasn't fully tested for all combinations of module and configs, and causes a compile failure: WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_ah.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_alloc.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_cmd.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_cq.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_db.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_hem.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_mr.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_pd.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_qp.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_restrack.o see include/linux/module.h for more information WARNING: modpost: missing MODULE_LICENSE() in drivers/infiniband/hw/hns/hns_roce_srq.o see include/linux/module.h for more information ERROR: "hns_roce_bitmap_cleanup" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_bitmap_init" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_free_cmd_mailbox" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_alloc_cmd_mailbox" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_table_get" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_bitmap_alloc" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! ERROR: "hns_roce_table_find" [drivers/infiniband/hw/hns/hns_roce_srq.ko] undefined! The fix is to put the module sub components in the right line. Fixes: e9816ddf2a33 ("RDMA/hns: Cleanup unnecessary exported symbols") Reported-by: Stephen Rothwell Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index 1c5a20af6992..e105945b94a1 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -10,8 +10,8 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o ifdef CONFIG_INFINIBAND_HNS_HIP06 -hns-roce-hw-v1-objs := hns_roce_hw_v1.o -obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o $(hns-roce-objs) +hns-roce-hw-v1-objs := hns_roce_hw_v1.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o endif ifdef CONFIG_INFINIBAND_HNS_HIP08 -- cgit v1.2.3 From 50ba3c18a4e549ba6a5a4672dfb3eb30fcb7d570 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 30 Jun 2019 18:48:32 +0300 Subject: RDMA/mlx5: Use proper allocation API to get zeroed memory There is no need in custom memory zeroing, because it can be done by using kzalloc from the beginning. Signed-off-by: Leon Romanovsky Reviewed-by: Jason Gunthorpe Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 692b60898ee2..1af36497d54c 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -4726,7 +4726,7 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) int err = -ENOMEM; struct ib_udata uhw = {.inlen = 0, .outlen = 0}; - pprops = kmalloc(sizeof(*pprops), GFP_KERNEL); + pprops = kzalloc(sizeof(*pprops), GFP_KERNEL); if (!pprops) goto out; @@ -4740,7 +4740,6 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) goto out; } - memset(pprops, 0, sizeof(*pprops)); err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); if (err) { mlx5_ib_warn(dev, "query_port %d failed %d\n", -- cgit v1.2.3 From bcde9a83b13ede042fd76e4cf0b759b6d6c0abe9 Mon Sep 17 00:00:00 2001 From: Daniel Kranzdorf Date: Sun, 30 Jun 2019 17:53:02 +0300 Subject: RDMA/efa: Entropy in admin commands id Make admin commands id easier to distinguish by using relevant bits from the producer counter. This allows us to differentiate admin commands with the same producer index (happens after admin queue overlap), which is helpful when debugging. Signed-off-by: Daniel Kranzdorf Reviewed-by: Firas JahJah Reviewed-by: Yossi Leybovich Signed-off-by: Gal Pressman Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_com.c | 44 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 21 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index ec04ced9fd2b..2cb42484b0f8 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -278,36 +278,34 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq, struct efa_comp_ctx *comp_ctx) { - u16 comp_id = comp_ctx->user_cqe->acq_common_descriptor.command & - EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; + u16 cmd_id = comp_ctx->user_cqe->acq_common_descriptor.command & + EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; + u16 ctx_id = cmd_id & (aq->depth - 1); - ibdev_dbg(aq->efa_dev, "Putting completion command_id %d\n", comp_id); + ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); comp_ctx->occupied = 0; - efa_com_dealloc_ctx_id(aq, comp_id); + efa_com_dealloc_ctx_id(aq, ctx_id); } static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq, - u16 command_id, bool capture) + u16 cmd_id, bool capture) { - if (command_id >= aq->depth) { - ibdev_err(aq->efa_dev, - "command id is larger than the queue size. cmd_id: %u queue size %d\n", - command_id, aq->depth); - return NULL; - } + u16 ctx_id = cmd_id & (aq->depth - 1); - if (aq->comp_ctx[command_id].occupied && capture) { - ibdev_err(aq->efa_dev, "Completion context is occupied\n"); + if (aq->comp_ctx[ctx_id].occupied && capture) { + ibdev_err(aq->efa_dev, + "Completion context for command_id %#x is occupied\n", + cmd_id); return NULL; } if (capture) { - aq->comp_ctx[command_id].occupied = 1; - ibdev_dbg(aq->efa_dev, "Taking completion ctxt command_id %d\n", - command_id); + aq->comp_ctx[ctx_id].occupied = 1; + ibdev_dbg(aq->efa_dev, + "Take completion ctxt for command_id %#x\n", cmd_id); } - return &aq->comp_ctx[command_id]; + return &aq->comp_ctx[ctx_id]; } static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, @@ -318,6 +316,7 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu { struct efa_comp_ctx *comp_ctx; u16 queue_size_mask; + u16 cmd_id; u16 ctx_id; u16 pi; @@ -326,13 +325,16 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu ctx_id = efa_com_alloc_ctx_id(aq); + /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ + cmd_id = ctx_id & queue_size_mask; + cmd_id |= aq->sq.pc & ~queue_size_mask; + cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; + + cmd->aq_common_descriptor.command_id = cmd_id; cmd->aq_common_descriptor.flags |= aq->sq.phase & EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; - cmd->aq_common_descriptor.command_id |= ctx_id & - EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; - - comp_ctx = efa_com_get_comp_ctx(aq, ctx_id, true); + comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true); if (!comp_ctx) { efa_com_dealloc_ctx_id(aq, ctx_id); return ERR_PTR(-EINVAL); -- cgit v1.2.3 From d14133dd41614aaaac1fa0505c7dab01f4211d2c Mon Sep 17 00:00:00 2001 From: Mark Zhang Date: Tue, 2 Jul 2019 13:02:36 +0300 Subject: IB/mlx5: Support set qp counter Support bind a qp with counter. If counter is null then bind the qp to the default counter. Different QP state has different operation: - RESET: Set the counter field so that it will take effective during RST2INIT change; - RTS: Issue an RTS2RTS change to update the QP counter; - Other: Set the counter field and mark the counter_pending flag, when QP is moved to RTS state and this flag is set, then issue an RTS2RTS modification to update the counter. Signed-off-by: Mark Zhang Reviewed-by: Majd Dibbiny Acked-by: Saeed Mahameed Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 6 +++ drivers/infiniband/hw/mlx5/qp.c | 76 +++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 7373e9da0919..c482f19958b3 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -439,6 +439,10 @@ struct mlx5_ib_qp { u32 flags_en; /* storage for qp sub type when core qp type is IB_QPT_DRIVER */ enum ib_qp_type qp_sub_type; + /* A flag to indicate if there's a new counter is configured + * but not take effective + */ + u32 counter_pending; }; struct mlx5_ib_cq_buf { @@ -1468,4 +1472,6 @@ void mlx5_ib_put_xlt_emergency_page(void); int bfregn_to_uar_index(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, u32 bfregn, bool dyn_bfreg); + +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); #endif /* MLX5_IB_H */ diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 8b7a60ada92c..2a97619ed603 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include "mlx5_ib.h" #include "ib_rep.h" @@ -3380,6 +3381,35 @@ static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev, return tx_port_affinity; } +static int __mlx5_ib_qp_set_counter(struct ib_qp *qp, + struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(qp->device); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_qp_context context = {}; + struct mlx5_ib_port *mibport = NULL; + struct mlx5_ib_qp_base *base; + u32 set_id; + + if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) + return 0; + + if (counter) { + set_id = counter->id; + } else { + mibport = &dev->port[mqp->port - 1]; + set_id = mibport->cnts.set_id; + } + + base = &mqp->trans_qp.base; + context.qp_counter_set_usr_page &= cpu_to_be32(0xffffff); + context.qp_counter_set_usr_page |= cpu_to_be32(set_id << 24); + return mlx5_core_qp_modify(dev->mdev, + MLX5_CMD_OP_RTS2RTS_QP, + MLX5_QP_OPTPAR_COUNTER_SET_ID, + &context, &base->mqp); +} + static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, @@ -3433,6 +3463,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, struct mlx5_ib_port *mibport = NULL; enum mlx5_qp_state mlx5_cur, mlx5_new; enum mlx5_qp_optpar optpar; + u32 set_id = 0; int mlx5_st; int err; u16 op; @@ -3595,8 +3626,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, port_num = 0; mibport = &dev->port[port_num]; + if (ibqp->counter) + set_id = ibqp->counter->id; + else + set_id = mibport->cnts.set_id; context->qp_counter_set_usr_page |= - cpu_to_be32((u32)(mibport->cnts.set_id) << 24); + cpu_to_be32(set_id << 24); } if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) @@ -3624,7 +3659,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, raw_qp_param.operation = op; if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - raw_qp_param.rq_q_ctr_id = mibport->cnts.set_id; + raw_qp_param.rq_q_ctr_id = set_id; raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID; } @@ -3701,6 +3736,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, qp->db.db[MLX5_SND_DBR] = 0; } + if ((new_state == IB_QPS_RTS) && qp->counter_pending) { + err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter); + if (!err) + qp->counter_pending = 0; + } + out: kfree(context); return err; @@ -6435,3 +6476,34 @@ void mlx5_ib_drain_rq(struct ib_qp *qp) handle_drain_completion(cq, &rdrain, dev); } + +/** + * Bind a qp to a counter. If @counter is NULL then bind the qp to + * the default counter + */ +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter) +{ + struct mlx5_ib_qp *mqp = to_mqp(qp); + int err = 0; + + mutex_lock(&mqp->mutex); + if (mqp->state == IB_QPS_RESET) { + qp->counter = counter; + goto out; + } + + if (mqp->state == IB_QPS_RTS) { + err = __mlx5_ib_qp_set_counter(qp, counter); + if (!err) + qp->counter = counter; + + goto out; + } + + mqp->counter_pending = 1; + qp->counter = counter; + +out: + mutex_unlock(&mqp->mutex); + return err; +} -- cgit v1.2.3 From 318d535cefecf2227ab20ec6e2961f9830c96e9b Mon Sep 17 00:00:00 2001 From: Mark Zhang Date: Tue, 2 Jul 2019 13:02:37 +0300 Subject: IB/mlx5: Add counter set id as a parameter for mlx5_ib_query_q_counters() Add counter set id as a parameter so that this API can be used for querying any q counter. Signed-off-by: Mark Zhang Reviewed-by: Majd Dibbiny Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 1af36497d54c..ac09706b9407 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -5477,7 +5477,8 @@ static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, struct mlx5_ib_port *port, - struct rdma_hw_stats *stats) + struct rdma_hw_stats *stats, + u16 set_id) { int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out); void *out; @@ -5488,9 +5489,7 @@ static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, if (!out) return -ENOMEM; - ret = mlx5_core_query_q_counter(mdev, - port->cnts.set_id, 0, - out, outlen); + ret = mlx5_core_query_q_counter(mdev, set_id, 0, out, outlen); if (ret) goto free; @@ -5550,7 +5549,8 @@ static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, port->cnts.num_ext_ppcnt_counters; /* q_counters are per IB device, query the master mdev */ - ret = mlx5_ib_query_q_counters(dev->mdev, port, stats); + ret = mlx5_ib_query_q_counters(dev->mdev, port, stats, + port->cnts.set_id); if (ret) return ret; -- cgit v1.2.3 From 45842fc627c7f0e55ad41613b64055266c710ac7 Mon Sep 17 00:00:00 2001 From: Mark Zhang Date: Tue, 2 Jul 2019 13:02:38 +0300 Subject: IB/mlx5: Support statistic q counter configuration Add support for ib callbacks counter_bind_qp(), counter_unbind_qp() and counter_dealloc(). Signed-off-by: Mark Zhang Reviewed-by: Majd Dibbiny Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index ac09706b9407..2914eab7c480 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -5586,6 +5586,47 @@ done: return num_counters; } +static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, + struct ib_qp *qp) +{ + struct mlx5_ib_dev *dev = to_mdev(qp->device); + u16 cnt_set_id = 0; + int err; + + if (!counter->id) { + err = mlx5_cmd_alloc_q_counter(dev->mdev, + &cnt_set_id, + MLX5_SHARED_RESOURCE_UID); + if (err) + return err; + counter->id = cnt_set_id; + } + + err = mlx5_ib_qp_set_counter(qp, counter); + if (err) + goto fail_set_counter; + + return 0; + +fail_set_counter: + mlx5_core_dealloc_q_counter(dev->mdev, cnt_set_id); + counter->id = 0; + + return err; +} + +static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) +{ + return mlx5_ib_qp_set_counter(qp, NULL); +} + +static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + + return mlx5_core_dealloc_q_counter(dev->mdev, counter->id); +} + static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params) @@ -6509,6 +6550,9 @@ static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev) static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = { .alloc_hw_stats = mlx5_ib_alloc_hw_stats, .get_hw_stats = mlx5_ib_get_hw_stats, + .counter_bind_qp = mlx5_ib_counter_bind_qp, + .counter_unbind_qp = mlx5_ib_counter_unbind_qp, + .counter_dealloc = mlx5_ib_counter_dealloc, }; static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) -- cgit v1.2.3 From 18d422ce8ccf47c65b98c2ce9e1758d84c8434eb Mon Sep 17 00:00:00 2001 From: Mark Zhang Date: Tue, 2 Jul 2019 13:02:41 +0300 Subject: IB/mlx5: Add counter_alloc_stats() and counter_update_stats() support Add support for ib callback counter_alloc_stats() and counter_update_stats(). Signed-off-by: Mark Zhang Reviewed-by: Majd Dibbiny Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 2914eab7c480..7581571bd9cd 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -5586,6 +5586,27 @@ done: return num_counters; } +static struct rdma_hw_stats * +mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + struct mlx5_ib_port *port = &dev->port[counter->port - 1]; + + /* Q counters are in the beginning of all counters */ + return rdma_alloc_hw_stats_struct(port->cnts.names, + port->cnts.num_q_counters, + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} + +static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + struct mlx5_ib_port *port = &dev->port[counter->port - 1]; + + return mlx5_ib_query_q_counters(dev->mdev, port, + counter->stats, counter->id); +} + static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, struct ib_qp *qp) { @@ -6553,6 +6574,8 @@ static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = { .counter_bind_qp = mlx5_ib_counter_bind_qp, .counter_unbind_qp = mlx5_ib_counter_unbind_qp, .counter_dealloc = mlx5_ib_counter_dealloc, + .counter_alloc_stats = mlx5_ib_counter_alloc_stats, + .counter_update_stats = mlx5_ib_counter_update_stats, }; static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) -- cgit v1.2.3 From 6fafe560ee4ddb79b1c9d6c168d4f2e7eef2a037 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Mon, 24 Jun 2019 19:47:45 +0800 Subject: RDMA/hns: Bugfix for cleaning mtr It uses hns_roce_mtr_init in hns_roce_create_qp_common function. As a result, it should use hns_roce_mtr_cleanup function for cleaning mtr when destroying qp. Fixes: 8d18ad83f19b ("RDMA/hns: Fix bug when wqe num is larger than 16K") Signed-off-by: Xi Wang Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 20e6b5139ef4..4ae547d96cd4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4565,7 +4565,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, (hr_qp->ibqp.qp_type == IB_QPT_UD)) hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); - hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); if (udata) { struct hns_roce_ucontext *context = -- cgit v1.2.3 From 21b97f538765996ac76b3a99adf371d0da5450f9 Mon Sep 17 00:00:00 2001 From: chenglang Date: Mon, 24 Jun 2019 19:47:46 +0800 Subject: RDMA/hns: Fixup qp release bug Hip06 reserve 12 qps, Hip08 reserve 8 qps. When the QP is released, the chip model is not judged, and the Hip08 cannot release the qpn 8~12 Signed-off-by: Lang Cheng Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 1 + drivers/infiniband/hw/hns/hns_roce_qp.c | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 998431c39b8d..2189e528d32d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -1559,6 +1559,7 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->reserved_mrws = 1; caps->reserved_uars = 0; caps->reserved_cqs = 0; + caps->reserved_qps = 12; /* 2 SQP per port, six ports total 12 */ caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE; for (i = 0; i < caps->num_ports; i++) diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 4f693cded74e..e7bbd6d967cb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -263,7 +263,7 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; - if (base_qpn < SQP_NUM) + if (base_qpn < hr_dev->caps.reserved_qps) return; hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); @@ -1226,11 +1226,7 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) mutex_init(&qp_table->scc_mutex); xa_init(&hr_dev->qp_table_xa); - /* In hw v1, a port include two SQP, six ports total 12 */ - if (hr_dev->caps.max_sq_sg <= 2) - reserved_from_bot = SQP_NUM; - else - reserved_from_bot = hr_dev->caps.reserved_qps; + reserved_from_bot = hr_dev->caps.reserved_qps; ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, hr_dev->caps.num_qps - 1, reserved_from_bot, -- cgit v1.2.3 From f5662b4ddc9db20033f310694edbd3fc7041f6f2 Mon Sep 17 00:00:00 2001 From: Yangyang Li Date: Mon, 24 Jun 2019 19:47:47 +0800 Subject: RDMA/hns: Modify ba page size for cqe Currently, the depth of cq only supports 64K. According to the UM, the depth of cq is up to 4M, Therefore the ba page size of cqe was modified to support the maximum specification of cq depth. Signed-off-by: Yangyang Li Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 4ae547d96cd4..3559c0cf112d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1616,7 +1616,7 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->wqe_sq_hop_num = 2; caps->wqe_sge_hop_num = 1; caps->wqe_rq_hop_num = 2; - caps->cqe_ba_pg_sz = 0; + caps->cqe_ba_pg_sz = 6; caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; caps->srqwqe_ba_pg_sz = 0; -- cgit v1.2.3 From 726be12f5ca0a9b464e7d91add512071e4c224f6 Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Mon, 24 Jun 2019 19:47:48 +0800 Subject: RDMA/hns: Set reset flag when hw resetting When hw resetting, there is no response from hw when driver sending cmdq. If driver still send cmdq to hw, the reset process may be blocked. So reset flag should be set to intercept the cmdq command when driver receiving "notify down" signal. Signed-off-by: Lang Cheng Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 3559c0cf112d..7676ea07fdec 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6384,6 +6384,7 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) if (!hr_dev) return 0; + hr_dev->is_reset = true; hr_dev->active = false; hr_dev->dis_db = true; -- cgit v1.2.3 From e0222d18d628ce96f0def790ea26b6b7b3c18f78 Mon Sep 17 00:00:00 2001 From: Lijun Ou Date: Mon, 24 Jun 2019 19:47:49 +0800 Subject: RDMA/hns: Bugfix for calculating qp buffer size The buffer size of qp which used to allocate qp buffer space for storing sqwqe and rqwqe will be the length of buffer space. The kernel driver will use the buffer address and the same size to get the user memory. The same size named buff_size of qp. According the algorithm of calculating, The size of the two is not equal when users set the max sge of sq. Fixes: b28ca7cceff8 ("RDMA/hns: Limit extend sq sge num") Signed-off-by: Lijun Ou Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_qp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index e7bbd6d967cb..c10960267f00 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -387,8 +387,8 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, hr_qp->sq.wqe_shift), PAGE_SIZE); } else { page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - hr_qp->sge.sge_cnt = - max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num); + hr_qp->sge.sge_cnt = ex_sge_num ? + max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size) + HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << -- cgit v1.2.3 From fd7dd8bc8784bc96213781f1bfb454ae1c13e79f Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Mon, 24 Jun 2019 19:47:50 +0800 Subject: RDMA/hns: Use %pK format pointer print The format specifier \"%p\" can leak kernel addresses. Use \"%pK\" instead. Signed-off-by: Lang Cheng Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_hw_v1.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 2189e528d32d..1a2c7dad2a0d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -3903,7 +3903,8 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, */ dma_rmb(); - dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe, + dev_dbg(dev, "aeqe = %pK, aeqe->asyn.event_type = 0x%lx\n", + aeqe, roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); -- cgit v1.2.3 From ec5bc2cc69b4fc494e04d10fc5226f6f9cf67c56 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Mon, 24 Jun 2019 19:47:51 +0800 Subject: RDMA/hns: Fixs hw access invalid dma memory error When smmu is enable, if execute the perftest command and then use 'kill -9' to exit, follow this operation repeatedly, the kernel will have a high probability to print the following smmu event: arm-smmu-v3 arm-smmu-v3.1.auto: event 0x10 received: arm-smmu-v3 arm-smmu-v3.1.auto: 0x00007d0000000010 arm-smmu-v3 arm-smmu-v3.1.auto: 0x0000020900000080 arm-smmu-v3 arm-smmu-v3.1.auto: 0x00000000f47cf000 arm-smmu-v3 arm-smmu-v3.1.auto: 0x00000000f47cf000 This is because the hw will periodically refresh the qpc cache until the next reset. This patch fixed it by removing the action that release qpc memory in the 'hns_roce_qp_free' function. Fixes: 9a4435375cd1 ("IB/hns: Add driver files for hns RoCE driver") Signed-off-by: Xi Wang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_qp.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index c10960267f00..e0424029b058 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -254,7 +254,6 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); - hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); } } -- cgit v1.2.3 From 617cf24f9f9efcd5e2f0258e7da10b8a2447662b Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Mon, 24 Jun 2019 19:47:52 +0800 Subject: RDMA/hns: Clean up unnecessary variable initialization Here Clean up unnecessary initial value for some variable. Signed-off-by: Lang Cheng Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/hns/hns_roce_cmd.c | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 4 ++-- drivers/infiniband/hw/hns/hns_roce_main.c | 2 +- drivers/infiniband/hw/hns/hns_roce_pd.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index b83d5bd92329..0cd09bf4d7ea 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -161,7 +161,7 @@ static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, unsigned long in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { - int ret = 0; + int ret; down(&hr_dev->cmd.event_sem); ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 7676ea07fdec..b76e3beeafb8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2018,7 +2018,7 @@ static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, unsigned long timeout) { struct device *dev = hr_dev->dev; - unsigned long end = 0; + unsigned long end; u32 status; end = msecs_to_jiffies(timeout) + jiffies; @@ -3016,7 +3016,7 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, { struct device *dev = hr_dev->dev; struct hns_roce_cmd_mailbox *mailbox; - int ret = 0; + int ret; u16 op = 0xff; if (!hns_roce_check_whether_mhop(hr_dev, table->type)) diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 9f83acec6001..1e4ba48f5613 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -309,7 +309,7 @@ static int hns_roce_modify_port(struct ib_device *ib_dev, u8 port_num, int mask, static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { - int ret = 0; + int ret; struct hns_roce_ucontext *context = to_hr_ucontext(uctx); struct hns_roce_ib_alloc_ucontext_resp resp = {}; struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 920ca76b5db1..912b89b4da34 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -92,7 +92,7 @@ void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) { struct resource *res; - int ret = 0; + int ret; /* Using bitmap to manager UAR index */ ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->logic_idx); -- cgit v1.2.3 From b6142608e8069dda26398e65b0a14eda6ca4282d Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Mon, 1 Jul 2019 21:14:02 +0300 Subject: IB/mlx5: Implement VHCA tunnel mechanism in DEVX This mechanism will allow function-A to perform operations "on behalf" of function-B via tunnel object. Function-A will have privileges for creating and using this tunnel object. For example, in the device emulation feature presented in Bluefield-1 SoC, using device emulation capability, one can present NVMe function to the host OS. Since the NVMe function doesn't have a normal command interface to the HCA HW, here is a need to create a channel that will be able to issue commands "on behalf" of this function. This channel is the VHCA_TUNNEL general object. The emulation software will create this tunnel for every managed function and issue commands via devx general cmd interface using the appropriate tunnel ID. When devX context will receive a command with non-zero vhca_tunnel_id, it will pass the command as-is down to the HCA. All the validation, security and resource tracking of the commands and the created tunneled objects is in the responsibility of the HCA FW. When a VHCA_TUNNEL object destroyed, the device will issue an internal FLR (function level reset) to the emulated function associated with this tunnel. This will destroy all the created resources using the tunnel mechanism. Signed-off-by: Max Gurtovoy Reviewed-by: Yishai Hadas Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/devx.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 931f587dfb8f..2aa833417492 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -715,12 +715,16 @@ static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in) return c->devx_uid; } -static bool devx_is_general_cmd(void *in) + +static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev) { u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); - if (opcode >= MLX5_CMD_OP_GENERAL_START && - opcode < MLX5_CMD_OP_GENERAL_END) + /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */ + if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) && + MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) || + (opcode >= MLX5_CMD_OP_GENERAL_START && + opcode < MLX5_CMD_OP_GENERAL_END)) return true; switch (opcode) { @@ -846,7 +850,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( return uid; /* Only white list of some general HCA commands are allowed for this method. */ - if (!devx_is_general_cmd(cmd_in)) + if (!devx_is_general_cmd(cmd_in, dev)) return -EINVAL; cmd_out = uverbs_zalloc(attrs, cmd_out_len); @@ -1169,6 +1173,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( u32 obj_id; u16 opcode; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1259,6 +1266,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( int err; int uid; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1301,6 +1311,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( int uid; struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1406,6 +1419,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( struct devx_async_cmd_event_file *ev_file; struct devx_async_data *async_data; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; -- cgit v1.2.3 From 89705e92700170888236555fe91b45e4c1bb0985 Mon Sep 17 00:00:00 2001 From: Danit Goldberg Date: Fri, 5 Jul 2019 19:21:57 +0300 Subject: IB/mlx5: Report correctly tag matching rendezvous capability Userspace expects the IB_TM_CAP_RC bit to indicate that the device supports RC transport tag matching with rendezvous offload. However the firmware splits this into two capabilities for eager and rendezvous tag matching. Only if the FW supports both modes should userspace be told the tag matching capability is available. Cc: # 4.13 Fixes: eb761894351d ("IB/mlx5: Fill XRQ capabilities") Signed-off-by: Danit Goldberg Reviewed-by: Yishai Hadas Reviewed-by: Artemy Kovalyov Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 7581571bd9cd..56d4b1e9dd23 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1046,15 +1046,19 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } if (MLX5_CAP_GEN(mdev, tag_matching)) { - props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE; props->tm_caps.max_num_tags = (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1; - props->tm_caps.flags = IB_TM_CAP_RC; props->tm_caps.max_ops = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); props->tm_caps.max_sge = MLX5_TM_MAX_SGE; } + if (MLX5_CAP_GEN(mdev, tag_matching) && + MLX5_CAP_GEN(mdev, rndv_offload_rc)) { + props->tm_caps.flags = IB_TM_CAP_RNDV_RC; + props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE; + } + if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) { props->cq_caps.max_cq_moderation_count = MLX5_MAX_CQ_COUNT; -- cgit v1.2.3 From 96e2fd733b9af304edcf30ef922dcaa4d3e1bfa6 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Mon, 8 Jul 2019 13:59:05 +0300 Subject: RDMA/mlx5: Set RDMA DIM to be enabled by default Enable RDMA DIM by default for better user experience. Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/mlx5/main.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/infiniband/hw') diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 56d4b1e9dd23..c2a5780cb394 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -6428,6 +6428,8 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) mutex_init(&dev->lb.mutex); + dev->ib_dev.use_cq_dim = true; + return 0; } -- cgit v1.2.3