From: Devesh Sharma <devesh.sharma@broadcom.com>
To: linux-rdma@vger.kernel.org
Cc: Devesh Sharma <devesh.sharma@broadcom.com>
Subject: [PATCH V4 rdma-core 4/5] bnxt_re/lib: Use separate indices for shadow queue
Date: Thu, 10 Jun 2021 16:19:09 +0530 [thread overview]
Message-ID: <20210610104910.1147756-5-devesh.sharma@broadcom.com> (raw)
In-Reply-To: <20210610104910.1147756-1-devesh.sharma@broadcom.com>
[-- Attachment #1: Type: text/plain, Size: 12836 bytes --]
The shadow queue is used for wrid and flush wqe management.
The indices used in this queue are independent to what is
actually used by hardware. Thus, detaching the shadow queue
indices from hardware queue indices. This even more useful
when the hardware queue indices has alignment other than wqe
boundary.
Signed-off-by: Devesh Sharma <devesh.sharma@broadcom.com>
---
providers/bnxt_re/main.h | 20 ++++++
providers/bnxt_re/memory.h | 8 +--
providers/bnxt_re/verbs.c | 128 ++++++++++++++++++++++---------------
3 files changed, 101 insertions(+), 55 deletions(-)
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h
index 94d42958..ad660e1a 100644
--- a/providers/bnxt_re/main.h
+++ b/providers/bnxt_re/main.h
@@ -437,4 +437,24 @@ static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq)
if (!cq->cqq.head)
cq->phase = (~cq->phase & BNXT_RE_BCQE_PH_MASK);
}
+
+static inline void *bnxt_re_get_swqe(struct bnxt_re_joint_queue *jqq,
+ uint32_t *wqe_idx)
+{
+ if (wqe_idx)
+ *wqe_idx = jqq->start_idx;
+ return &jqq->swque[jqq->start_idx];
+}
+
+static inline void bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue *jqq,
+ uint32_t idx)
+{
+ jqq->start_idx = jqq->swque[idx].next_idx;
+}
+
+static inline void bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue *jqq,
+ uint32_t idx)
+{
+ jqq->last_idx = jqq->swque[idx].next_idx;
+}
#endif
diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h
index 75564c43..5bcdef9a 100644
--- a/providers/bnxt_re/memory.h
+++ b/providers/bnxt_re/memory.h
@@ -97,14 +97,14 @@ static inline uint32_t bnxt_re_incr(uint32_t val, uint32_t max)
return (++val & (max - 1));
}
-static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que)
+static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint8_t cnt)
{
- que->tail = bnxt_re_incr(que->tail, que->depth);
+ que->tail = (que->tail + cnt) & (que->depth - 1);
}
-static inline void bnxt_re_incr_head(struct bnxt_re_queue *que)
+static inline void bnxt_re_incr_head(struct bnxt_re_queue *que, uint8_t cnt)
{
- que->head = bnxt_re_incr(que->head, que->depth);
+ que->head = (que->head + cnt) & (que->depth - 1);
}
#endif
diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c
index e0e6e045..268f443c 100644
--- a/providers/bnxt_re/verbs.c
+++ b/providers/bnxt_re/verbs.c
@@ -247,10 +247,12 @@ static uint8_t bnxt_re_poll_err_scqe(struct bnxt_re_qp *qp,
struct bnxt_re_wrid *swrid;
struct bnxt_re_psns *spsn;
struct bnxt_re_cq *scq;
- uint32_t head = sq->head;
uint8_t status;
+ uint32_t head;
scq = to_bnxt_re_cq(qp->ibvqp.send_cq);
+
+ head = qp->jsqq->last_idx;
cntx = to_bnxt_re_context(scq->ibvcq.context);
swrid = &qp->jsqq->swque[head];
spsn = swrid->psns;
@@ -267,7 +269,8 @@ static uint8_t bnxt_re_poll_err_scqe(struct bnxt_re_qp *qp,
BNXT_RE_PSNS_OPCD_MASK;
ibvwc->byte_len = 0;
- bnxt_re_incr_head(sq);
+ bnxt_re_incr_head(sq, swrid->slots);
+ bnxt_re_jqq_mod_last(qp->jsqq, head);
if (qp->qpst != IBV_QPS_ERR)
qp->qpst = IBV_QPS_ERR;
@@ -287,13 +290,14 @@ static uint8_t bnxt_re_poll_success_scqe(struct bnxt_re_qp *qp,
struct bnxt_re_queue *sq = qp->jsqq->hwque;
struct bnxt_re_wrid *swrid;
struct bnxt_re_psns *spsn;
- uint32_t head = sq->head;
uint8_t pcqe = false;
uint32_t cindx;
+ uint32_t head;
+ head = qp->jsqq->last_idx;
swrid = &qp->jsqq->swque[head];
spsn = swrid->psns;
- cindx = le32toh(scqe->con_indx);
+ cindx = le32toh(scqe->con_indx) & (qp->cap.max_swr - 1);
if (!(swrid->sig & IBV_SEND_SIGNALED)) {
*cnt = 0;
@@ -313,8 +317,10 @@ static uint8_t bnxt_re_poll_success_scqe(struct bnxt_re_qp *qp,
*cnt = 1;
}
- bnxt_re_incr_head(sq);
- if (sq->head != cindx)
+ bnxt_re_incr_head(sq, swrid->slots);
+ bnxt_re_jqq_mod_last(qp->jsqq, head);
+
+ if (qp->jsqq->last_idx != cindx)
pcqe = true;
return pcqe;
@@ -352,23 +358,29 @@ static void bnxt_re_release_srqe(struct bnxt_re_srq *srq, int tag)
static int bnxt_re_poll_err_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
struct bnxt_re_bcqe *hdr, void *cqe)
{
+ struct bnxt_re_context *cntx;
+ struct bnxt_re_wrid *swque;
struct bnxt_re_queue *rq;
+ uint8_t status, cnt = 0;
struct bnxt_re_cq *rcq;
- struct bnxt_re_context *cntx;
- uint8_t status;
+ uint32_t head = 0;
rcq = to_bnxt_re_cq(qp->ibvqp.recv_cq);
cntx = to_bnxt_re_context(rcq->ibvcq.context);
if (!qp->srq) {
rq = qp->jrqq->hwque;
- ibvwc->wr_id = qp->jrqq->swque[rq->head].wrid;
+ head = qp->jrqq->last_idx;
+ swque = &qp->jrqq->swque[head];
+ ibvwc->wr_id = swque->wrid;
+ cnt = swque->slots;
} else {
struct bnxt_re_srq *srq;
int tag;
srq = qp->srq;
rq = srq->srqq;
+ cnt = 1;
tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK;
ibvwc->wr_id = srq->srwrid[tag].wrid;
bnxt_re_release_srqe(srq, tag);
@@ -387,7 +399,10 @@ static int bnxt_re_poll_err_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
ibvwc->wc_flags = 0;
if (qp->qptyp == IBV_QPT_UD)
ibvwc->src_qp = 0;
- bnxt_re_incr_head(rq);
+
+ if (!qp->srq)
+ bnxt_re_jqq_mod_last(qp->jrqq, head);
+ bnxt_re_incr_head(rq, cnt);
if (!qp->srq) {
pthread_spin_lock(&cntx->fqlock);
@@ -417,14 +432,20 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
struct ibv_wc *ibvwc,
struct bnxt_re_bcqe *hdr, void *cqe)
{
- struct bnxt_re_queue *rq;
- struct bnxt_re_rc_cqe *rcqe;
uint8_t flags, is_imm, is_rdma;
+ struct bnxt_re_rc_cqe *rcqe;
+ struct bnxt_re_wrid *swque;
+ struct bnxt_re_queue *rq;
+ uint32_t head = 0;
+ uint8_t cnt = 0;
rcqe = cqe;
if (!qp->srq) {
rq = qp->jrqq->hwque;
- ibvwc->wr_id = qp->jrqq->swque[rq->head].wrid;
+ head = qp->jrqq->last_idx;
+ swque = &qp->jrqq->swque[head];
+ ibvwc->wr_id = swque->wrid;
+ cnt = swque->slots;
} else {
struct bnxt_re_srq *srq;
int tag;
@@ -433,6 +454,7 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
rq = srq->srqq;
tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK;
ibvwc->wr_id = srq->srwrid[tag].wrid;
+ cnt = 1;
bnxt_re_release_srqe(srq, tag);
}
@@ -463,7 +485,9 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
if (qp->qptyp == IBV_QPT_UD)
bnxt_re_fill_ud_cqe(ibvwc, hdr, cqe);
- bnxt_re_incr_head(rq);
+ if (!qp->srq)
+ bnxt_re_jqq_mod_last(qp->jrqq, head);
+ bnxt_re_incr_head(rq, cnt);
}
static uint8_t bnxt_re_poll_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
@@ -575,7 +599,7 @@ static int bnxt_re_poll_one(struct bnxt_re_cq *cq, int nwc, struct ibv_wc *wc)
*qp_handle = 0x0ULL; /* mark cqe as read */
qp_handle = NULL;
}
- bnxt_re_incr_head(&cq->cqq);
+ bnxt_re_incr_head(&cq->cqq, 1);
bnxt_re_change_cq_phase(cq);
skipp_real:
if (cnt) {
@@ -592,21 +616,21 @@ skipp_real:
return dqed;
}
-static int bnxt_re_poll_flush_wcs(struct bnxt_re_queue *que,
- struct bnxt_re_wrid *wridp,
+static int bnxt_re_poll_flush_wcs(struct bnxt_re_joint_queue *jqq,
struct ibv_wc *ibvwc, uint32_t qpid,
int nwc)
{
+ uint8_t opcode = IBV_WC_RECV;
+ struct bnxt_re_queue *que;
struct bnxt_re_wrid *wrid;
struct bnxt_re_psns *psns;
- uint32_t cnt = 0, head;
- uint8_t opcode = IBV_WC_RECV;
+ uint32_t cnt = 0;
+ que = jqq->hwque;
while (nwc) {
if (bnxt_re_is_que_empty(que))
break;
- head = que->head;
- wrid = &wridp[head];
+ wrid = &jqq->swque[jqq->last_idx];
if (wrid->psns) {
psns = wrid->psns;
opcode = (le32toh(psns->opc_spsn) >>
@@ -621,7 +645,8 @@ static int bnxt_re_poll_flush_wcs(struct bnxt_re_queue *que,
ibvwc->byte_len = 0;
ibvwc->wc_flags = 0;
- bnxt_re_incr_head(que);
+ bnxt_re_jqq_mod_last(jqq, jqq->last_idx);
+ bnxt_re_incr_head(que, wrid->slots);
nwc--;
cnt++;
ibvwc++;
@@ -636,8 +661,7 @@ static int bnxt_re_poll_flush_wqes(struct bnxt_re_cq *cq,
int32_t nwc)
{
struct bnxt_re_fque_node *cur, *tmp;
- struct bnxt_re_wrid *wridp;
- struct bnxt_re_queue *que;
+ struct bnxt_re_joint_queue *jqq;
struct bnxt_re_qp *qp;
bool sq_list = false;
uint32_t polled = 0;
@@ -648,18 +672,15 @@ static int bnxt_re_poll_flush_wqes(struct bnxt_re_cq *cq,
if (sq_list) {
qp = container_of(cur, struct bnxt_re_qp,
snode);
- que = qp->jsqq->hwque;
- wridp = qp->jsqq->swque;
+ jqq = qp->jsqq;
} else {
qp = container_of(cur, struct bnxt_re_qp,
rnode);
- que = qp->jrqq->hwque;
- wridp = qp->jrqq->swque;
+ jqq = qp->jrqq;
}
- if (bnxt_re_is_que_empty(que))
+ if (bnxt_re_is_que_empty(jqq->hwque))
continue;
- polled += bnxt_re_poll_flush_wcs(que, wridp,
- ibvwc + polled,
+ polled += bnxt_re_poll_flush_wcs(jqq, ibvwc + polled,
qp->qpid,
nwc - polled);
if (!(nwc - polled))
@@ -1165,14 +1186,17 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid,
psns_ext->st_slot_idx = 0;
}
-static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, struct ibv_send_wr *wr,
- uint32_t len, uint8_t sqsig)
+static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id,
+ uint32_t len, uint8_t sqsig, uint32_t st_idx,
+ uint8_t slots)
{
- wrid->wrid = wr->wr_id;
+ wrid->wrid = wr_id;
wrid->bytes = len;
wrid->sig = 0;
- if (wr->send_flags & IBV_SEND_SIGNALED || sqsig)
+ if (sqsig)
wrid->sig = IBV_SEND_SIGNALED;
+ wrid->st_slot_idx = st_idx;
+ wrid->slots = slots;
}
static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe,
@@ -1291,6 +1315,8 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
struct bnxt_re_bsqe *hdr;
int ret = 0, bytes = 0;
bool ring_db = false;
+ uint32_t swq_idx;
+ uint32_t sig;
void *sqe;
pthread_spin_lock(&sq->qlock);
@@ -1317,8 +1343,6 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
}
sqe = (void *)(sq->va + (sq->tail * sq->stride));
- wrid = &qp->jsqq->swque[sq->tail];
-
memset(sqe, 0, bnxt_re_get_sqe_sz());
hdr = sqe;
is_inline = bnxt_re_set_hdr_flags(hdr, wr->send_flags,
@@ -1366,9 +1390,12 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
break;
}
- bnxt_re_fill_wrid(wrid, wr, bytes, qp->cap.sqsig);
+ wrid = bnxt_re_get_swqe(qp->jsqq, &swq_idx);
+ sig = ((wr->send_flags & IBV_SEND_SIGNALED) || qp->cap.sqsig);
+ bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, sig, sq->tail, 1);
bnxt_re_fill_psns(qp, wrid, wr->opcode, bytes);
- bnxt_re_incr_tail(sq);
+ bnxt_re_jqq_mod_start(qp->jsqq, swq_idx);
+ bnxt_re_incr_tail(sq, 1);
qp->wqe_cnt++;
wr = wr->next;
ring_db = true;
@@ -1395,16 +1422,14 @@ bad_wr:
}
static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr,
- void *rqe)
+ void *rqe, uint32_t idx)
{
struct bnxt_re_brqe *hdr = rqe;
- struct bnxt_re_wrid *wrid;
struct bnxt_re_sge *sge;
int wqe_sz, len;
uint32_t hdrval;
sge = (rqe + bnxt_re_get_rqe_hdr_sz());
- wrid = &qp->jrqq->swque[qp->jrqq->hwque->tail];
len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false);
wqe_sz = wr->num_sge + (bnxt_re_get_rqe_hdr_sz() >> 4); /* 16B align */
@@ -1416,12 +1441,7 @@ static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr,
hdrval = BNXT_RE_WR_OPCD_RECV;
hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT);
hdr->rsv_ws_fl_wt = htole32(hdrval);
- hdr->wrid = htole32(qp->jrqq->hwque->tail);
-
- /* Fill wrid */
- wrid->wrid = wr->wr_id;
- wrid->bytes = len; /* N.A. for RQE */
- wrid->sig = 0; /* N.A. for RQE */
+ hdr->wrid = htole32(idx);
return len;
}
@@ -1431,6 +1451,8 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
{
struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp);
struct bnxt_re_queue *rq = qp->jrqq->hwque;
+ struct bnxt_re_wrid *swque;
+ uint32_t swq_idx;
void *rqe;
int ret;
@@ -1452,14 +1474,18 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
rqe = (void *)(rq->va + (rq->tail * rq->stride));
memset(rqe, 0, bnxt_re_get_rqe_sz());
- ret = bnxt_re_build_rqe(qp, wr, rqe);
+ swque = bnxt_re_get_swqe(qp->jrqq, &swq_idx);
+ ret = bnxt_re_build_rqe(qp, wr, rqe, swq_idx);
if (ret < 0) {
pthread_spin_unlock(&rq->qlock);
*bad = wr;
return ENOMEM;
}
- bnxt_re_incr_tail(rq);
+ swque = bnxt_re_get_swqe(qp->jrqq, NULL);
+ bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, 1);
+ bnxt_re_jqq_mod_start(qp->jrqq, swq_idx);
+ bnxt_re_incr_tail(rq, 1);
wr = wr->next;
bnxt_re_ring_rq_db(qp);
}
@@ -1667,7 +1693,7 @@ int bnxt_re_post_srq_recv(struct ibv_srq *ibvsrq, struct ibv_recv_wr *wr,
}
srq->start_idx = srq->srwrid[srq->start_idx].next_idx;
- bnxt_re_incr_tail(rq);
+ bnxt_re_incr_tail(rq, 1);
wr = wr->next;
bnxt_re_ring_srq_db(srq);
count++;
--
2.25.1
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4212 bytes --]
next prev parent reply other threads:[~2021-06-10 10:50 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-06-10 10:49 [PATCH V4 rdma-core 0/5] Broadcom's user library update Devesh Sharma
2021-06-10 10:49 ` [PATCH V4 rdma-core 1/5] Update kernel headers Devesh Sharma
2021-06-10 12:43 ` Haakon Bugge
2021-06-10 15:34 ` Devesh Sharma
2021-06-11 7:01 ` Haakon Bugge
2021-06-10 10:49 ` [PATCH V4 rdma-core 2/5] bnxt_re/lib: Read wqe mode from the driver Devesh Sharma
2021-06-10 10:49 ` [PATCH V4 rdma-core 3/5] bnxt_re/lib: add a function to initialize software queue Devesh Sharma
2021-06-10 12:50 ` Haakon Bugge
2021-06-10 15:36 ` Devesh Sharma
2021-06-11 6:59 ` Haakon Bugge
2021-06-10 10:49 ` Devesh Sharma [this message]
2021-06-10 10:49 ` [PATCH V4 rdma-core 5/5] bnxt_re/lib: Move hardware queue to 16B aligned indices Devesh Sharma
2021-06-10 13:16 ` Haakon Bugge
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210610104910.1147756-5-devesh.sharma@broadcom.com \
--to=devesh.sharma@broadcom.com \
--cc=linux-rdma@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.