From faa63656fc361e78ebccb288c0be638a6a4ffe35 Mon Sep 17 00:00:00 2001 From: Yixing Liu Date: Sat, 4 Mar 2023 17:15:54 +0800 Subject: RDMA/hns: Add new command to support query vf caps The current resource query for vf caps is driven by the driver, which is unreasonable. This patch adds a new command HNS_ROCE_OPC_QUERY_VF_CAPS_NUM to support obtaining vf caps information from firmware. Signed-off-by: Yixing Liu Signed-off-by: Haoyue Xu Link: https://lore.kernel.org/r/20230304091555.2241298-2-xuhaoyue1@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 189 +++++++++-------------------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 31 +---- 2 files changed, 56 insertions(+), 164 deletions(-) (limited to 'drivers/infiniband/hw/hns') diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index dbf97fe5948f..84f1167de1d9 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1960,100 +1960,6 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev) return hns_roce_cmq_send(hr_dev, &desc, 1); } -/* Use default caps when hns_roce_query_pf_caps() failed or init VF profile */ -static void set_default_caps(struct hns_roce_dev *hr_dev) -{ - struct hns_roce_caps *caps = &hr_dev->caps; - - caps->num_qps = HNS_ROCE_V2_MAX_QP_NUM; - caps->max_wqes = HNS_ROCE_V2_MAX_WQE_NUM; - caps->num_cqs = HNS_ROCE_V2_MAX_CQ_NUM; - caps->num_srqs = HNS_ROCE_V2_MAX_SRQ_NUM; - caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; - caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; - caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; - caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; - - caps->num_uars = HNS_ROCE_V2_UAR_NUM; - caps->phy_num_uars = HNS_ROCE_V2_PHY_UAR_NUM; - caps->num_aeq_vectors = HNS_ROCE_V2_AEQE_VEC_NUM; - caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM; - caps->num_comp_vectors = 0; - - caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; - caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; - caps->qpc_timer_bt_num = HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM; - caps->cqc_timer_bt_num = HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM; - - caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; - caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; - caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; - caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; - caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; - caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; - caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; - caps->srqc_entry_sz = HNS_ROCE_V2_SRQC_ENTRY_SZ; - caps->mtpt_entry_sz = HNS_ROCE_V2_MTPT_ENTRY_SZ; - caps->idx_entry_sz = HNS_ROCE_V2_IDX_ENTRY_SZ; - caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; - caps->reserved_lkey = 0; - caps->reserved_pds = 0; - caps->reserved_mrws = 1; - caps->reserved_uars = 0; - caps->reserved_cqs = 0; - caps->reserved_srqs = 0; - caps->reserved_qps = HNS_ROCE_V2_RSV_QPS; - - caps->qpc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->srqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->cqc_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->mpt_hop_num = HNS_ROCE_CONTEXT_HOP_NUM; - caps->sccc_hop_num = HNS_ROCE_SCCC_HOP_NUM; - - caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; - caps->wqe_sq_hop_num = HNS_ROCE_SQWQE_HOP_NUM; - caps->wqe_sge_hop_num = HNS_ROCE_EXT_SGE_HOP_NUM; - caps->wqe_rq_hop_num = HNS_ROCE_RQWQE_HOP_NUM; - caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; - caps->srqwqe_hop_num = HNS_ROCE_SRQWQE_HOP_NUM; - caps->idx_hop_num = HNS_ROCE_IDX_HOP_NUM; - caps->chunk_sz = HNS_ROCE_V2_TABLE_CHUNK_SIZE; - - caps->flags = HNS_ROCE_CAP_FLAG_REREG_MR | - HNS_ROCE_CAP_FLAG_ROCE_V1_V2 | - HNS_ROCE_CAP_FLAG_CQ_RECORD_DB | - HNS_ROCE_CAP_FLAG_QP_RECORD_DB; - - caps->pkey_table_len[0] = 1; - caps->ceqe_depth = HNS_ROCE_V2_COMP_EQE_NUM; - caps->aeqe_depth = HNS_ROCE_V2_ASYNC_EQE_NUM; - caps->local_ca_ack_delay = 0; - caps->max_mtu = IB_MTU_4096; - - caps->max_srq_wrs = HNS_ROCE_V2_MAX_SRQ_WR; - caps->max_srq_sges = HNS_ROCE_V2_MAX_SRQ_SGE; - - caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC | HNS_ROCE_CAP_FLAG_MW | - HNS_ROCE_CAP_FLAG_SRQ | HNS_ROCE_CAP_FLAG_FRMR | - HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL; - - caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM; - - if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { - caps->flags |= HNS_ROCE_CAP_FLAG_STASH | - HNS_ROCE_CAP_FLAG_DIRECT_WQE | - HNS_ROCE_CAP_FLAG_XRC; - caps->max_sq_inline = HNS_ROCE_V3_MAX_SQ_INLINE; - } else { - caps->max_sq_inline = HNS_ROCE_V2_MAX_SQ_INLINE; - - /* The following configuration are only valid for HIP08 */ - caps->qpc_sz = HNS_ROCE_V2_QPC_SZ; - caps->sccc_sz = HNS_ROCE_V2_SCCC_SZ; - caps->cqe_sz = HNS_ROCE_V2_CQE_SIZE; - } -} - static void calc_pg_sz(u32 obj_num, u32 obj_size, u32 hop_num, u32 ctx_bt_num, u32 *buf_page_size, u32 *bt_page_size, u32 hem_type) { @@ -2239,7 +2145,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) set_hem_page_size(hr_dev); } -static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) +static int hns_roce_query_caps(struct hns_roce_dev *hr_dev) { struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM]; struct hns_roce_caps *caps = &hr_dev->caps; @@ -2248,15 +2154,17 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) struct hns_roce_query_pf_caps_c *resp_c; struct hns_roce_query_pf_caps_d *resp_d; struct hns_roce_query_pf_caps_e *resp_e; + enum hns_roce_opcode_type cmd; int ctx_hop_num; int pbl_hop_num; int ret; int i; + cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM : + HNS_ROCE_OPC_QUERY_PF_CAPS_NUM; + for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) { - hns_roce_cmq_setup_basic_desc(&desc[i], - HNS_ROCE_OPC_QUERY_PF_CAPS_NUM, - true); + hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true); if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1)) desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT); else @@ -2273,35 +2181,33 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data; resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data; - caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; - caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); - caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); - caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); + caps->local_ca_ack_delay = resp_a->local_ca_ack_delay; + caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg); + caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); + caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); - caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); + caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges); - caps->num_aeq_vectors = resp_a->num_aeq_vectors; - caps->num_other_vectors = resp_a->num_other_vectors; - caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; - caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; - caps->cqe_sz = resp_a->cqe_sz; - - caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; - caps->irrl_entry_sz = resp_b->irrl_entry_sz; - caps->trrl_entry_sz = resp_b->trrl_entry_sz; - caps->cqc_entry_sz = resp_b->cqc_entry_sz; - caps->srqc_entry_sz = resp_b->srqc_entry_sz; - caps->idx_entry_sz = resp_b->idx_entry_sz; - caps->sccc_sz = resp_b->sccc_sz; - caps->max_mtu = resp_b->max_mtu; - caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); - caps->min_cqes = resp_b->min_cqes; - caps->min_wqes = resp_b->min_wqes; - caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); - caps->pkey_table_len[0] = resp_b->pkey_table_len; - caps->phy_num_uars = resp_b->phy_num_uars; - ctx_hop_num = resp_b->ctx_hop_num; - pbl_hop_num = resp_b->pbl_hop_num; + caps->num_aeq_vectors = resp_a->num_aeq_vectors; + caps->num_other_vectors = resp_a->num_other_vectors; + caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; + caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; + + caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; + caps->irrl_entry_sz = resp_b->irrl_entry_sz; + caps->trrl_entry_sz = resp_b->trrl_entry_sz; + caps->cqc_entry_sz = resp_b->cqc_entry_sz; + caps->srqc_entry_sz = resp_b->srqc_entry_sz; + caps->idx_entry_sz = resp_b->idx_entry_sz; + caps->sccc_sz = resp_b->sccc_sz; + caps->max_mtu = resp_b->max_mtu; + caps->min_cqes = resp_b->min_cqes; + caps->min_wqes = resp_b->min_wqes; + caps->page_size_cap = le32_to_cpu(resp_b->page_size_cap); + caps->pkey_table_len[0] = resp_b->pkey_table_len; + caps->phy_num_uars = resp_b->phy_num_uars; + ctx_hop_num = resp_b->ctx_hop_num; + pbl_hop_num = resp_b->pbl_hop_num; caps->num_pds = 1 << hr_reg_read(resp_c, PF_CAPS_C_NUM_PDS); @@ -2324,8 +2230,6 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->ceqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_CEQ_DEPTH); caps->num_comp_vectors = hr_reg_read(resp_d, PF_CAPS_D_NUM_CEQS); caps->aeqe_depth = 1 << hr_reg_read(resp_d, PF_CAPS_D_AEQ_DEPTH); - caps->default_aeq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST); - caps->default_ceq_arm_st = hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST); caps->reserved_pds = hr_reg_read(resp_d, PF_CAPS_D_RSV_PDS); caps->num_uars = 1 << hr_reg_read(resp_d, PF_CAPS_D_NUM_UARS); caps->reserved_qps = hr_reg_read(resp_d, PF_CAPS_D_RSV_QPS); @@ -2336,10 +2240,6 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->reserved_cqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_CQS); caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS); caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS); - caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); - caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); - caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); - caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); caps->qpc_hop_num = ctx_hop_num; caps->sccc_hop_num = ctx_hop_num; @@ -2357,6 +2257,19 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) if (!(caps->page_size_cap & PAGE_SIZE)) caps->page_size_cap = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED; + if (!hr_dev->is_vf) { + caps->cqe_sz = resp_a->cqe_sz; + caps->qpc_sz = le16_to_cpu(resp_b->qpc_sz); + caps->default_aeq_arm_st = + hr_reg_read(resp_d, PF_CAPS_D_AEQ_ARM_ST); + caps->default_ceq_arm_st = + hr_reg_read(resp_d, PF_CAPS_D_CEQ_ARM_ST); + caps->default_ceq_max_cnt = le16_to_cpu(resp_e->ceq_max_cnt); + caps->default_ceq_period = le16_to_cpu(resp_e->ceq_period); + caps->default_aeq_max_cnt = le16_to_cpu(resp_e->aeq_max_cnt); + caps->default_aeq_period = le16_to_cpu(resp_e->aeq_period); + } + return 0; } @@ -2404,7 +2317,11 @@ static int hns_roce_v2_vf_profile(struct hns_roce_dev *hr_dev) hr_dev->func_num = 1; - set_default_caps(hr_dev); + ret = hns_roce_query_caps(hr_dev); + if (ret) { + dev_err(dev, "failed to query VF caps, ret = %d.\n", ret); + return ret; + } ret = hns_roce_query_vf_resource(hr_dev); if (ret) { @@ -2444,9 +2361,11 @@ static int hns_roce_v2_pf_profile(struct hns_roce_dev *hr_dev) return ret; } - ret = hns_roce_query_pf_caps(hr_dev); - if (ret) - set_default_caps(hr_dev); + ret = hns_roce_query_caps(hr_dev); + if (ret) { + dev_err(dev, "failed to query PF caps, ret = %d.\n", ret); + return ret; + } ret = hns_roce_query_pf_resource(hr_dev); if (ret) { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index af9d00225cdf..1b44d2434ab4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -35,43 +35,15 @@ #include -#define HNS_ROCE_V2_MAX_QP_NUM 0x1000 -#define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 -#define HNS_ROCE_V2_MAX_SRQ_WR 0x8000 -#define HNS_ROCE_V2_MAX_SRQ_SGE 64 -#define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 -#define HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM 0x100 -#define HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM 0x100 -#define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 -#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 -#define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64 -#define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 -#define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 -#define HNS_ROCE_V3_MAX_SQ_INLINE 0x400 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 -#define HNS_ROCE_V2_UAR_NUM 256 -#define HNS_ROCE_V2_PHY_UAR_NUM 1 +#define HNS_ROCE_V2_MTT_ENTRY_SZ 64 #define HNS_ROCE_V2_AEQE_VEC_NUM 1 #define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1 -#define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000 #define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000 -#define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 #define HNS_ROCE_V2_MAX_XRCD_NUM 0x1000000 #define HNS_ROCE_V2_RSV_XRCD_NUM 0 -#define HNS_ROCE_V2_MAX_QP_INIT_RDMA 128 -#define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128 -#define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 -#define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 -#define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 -#define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 -#define HNS_ROCE_V2_CQC_ENTRY_SZ 64 -#define HNS_ROCE_V2_SRQC_ENTRY_SZ 64 -#define HNS_ROCE_V2_MTPT_ENTRY_SZ 64 -#define HNS_ROCE_V2_MTT_ENTRY_SZ 64 -#define HNS_ROCE_V2_IDX_ENTRY_SZ 4 -#define HNS_ROCE_V2_SCCC_SZ 32 #define HNS_ROCE_V3_SCCC_SZ 64 #define HNS_ROCE_V3_GMV_ENTRY_SZ 32 @@ -232,6 +204,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_QUERY_FUNC_INFO = 0x8407, HNS_ROCE_OPC_QUERY_PF_CAPS_NUM = 0x8408, HNS_ROCE_OPC_CFG_ENTRY_SIZE = 0x8409, + HNS_ROCE_OPC_QUERY_VF_CAPS_NUM = 0x8410, HNS_ROCE_OPC_CFG_SGID_TB = 0x8500, HNS_ROCE_OPC_CFG_SMAC_TB = 0x8501, HNS_ROCE_OPC_POST_MB = 0x8504, -- cgit v1.2.3