324 lines
8.5 KiB
C
324 lines
8.5 KiB
C
|
// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
|
||
|
|
/*
|
||
|
|
* Copyright(c) 2018 Intel Corporation.
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
#include "hfi.h"
|
||
|
|
#include "trace.h"
|
||
|
|
#include "qp.h"
|
||
|
|
#include "opfn.h"
|
||
|
|
|
||
|
|
#define IB_BTHE_E BIT(IB_BTHE_E_SHIFT)
|
||
|
|
|
||
|
|
#define OPFN_CODE(code) BIT((code) - 1)
|
||
|
|
#define OPFN_MASK(code) OPFN_CODE(STL_VERBS_EXTD_##code)
|
||
|
|
|
||
|
|
struct hfi1_opfn_type {
|
||
|
|
bool (*request)(struct rvt_qp *qp, u64 *data);
|
||
|
|
bool (*response)(struct rvt_qp *qp, u64 *data);
|
||
|
|
bool (*reply)(struct rvt_qp *qp, u64 data);
|
||
|
|
void (*error)(struct rvt_qp *qp);
|
||
|
|
};
|
||
|
|
|
||
|
|
static struct hfi1_opfn_type hfi1_opfn_handlers[STL_VERBS_EXTD_MAX] = {
|
||
|
|
[STL_VERBS_EXTD_TID_RDMA] = {
|
||
|
|
.request = tid_rdma_conn_req,
|
||
|
|
.response = tid_rdma_conn_resp,
|
||
|
|
.reply = tid_rdma_conn_reply,
|
||
|
|
.error = tid_rdma_conn_error,
|
||
|
|
},
|
||
|
|
};
|
||
|
|
|
||
|
|
static struct workqueue_struct *opfn_wq;
|
||
|
|
|
||
|
|
static void opfn_schedule_conn_request(struct rvt_qp *qp);
|
||
|
|
|
||
|
|
static bool hfi1_opfn_extended(u32 bth1)
|
||
|
|
{
|
||
|
|
return !!(bth1 & IB_BTHE_E);
|
||
|
|
}
|
||
|
|
|
||
|
|
static void opfn_conn_request(struct rvt_qp *qp)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
struct ib_atomic_wr wr;
|
||
|
|
u16 mask, capcode;
|
||
|
|
struct hfi1_opfn_type *extd;
|
||
|
|
u64 data;
|
||
|
|
unsigned long flags;
|
||
|
|
int ret = 0;
|
||
|
|
|
||
|
|
trace_hfi1_opfn_state_conn_request(qp);
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
/*
|
||
|
|
* Exit if the extended bit is not set, or if nothing is requested, or
|
||
|
|
* if we have completed all requests, or if a previous request is in
|
||
|
|
* progress
|
||
|
|
*/
|
||
|
|
if (!priv->opfn.extended || !priv->opfn.requested ||
|
||
|
|
priv->opfn.requested == priv->opfn.completed || priv->opfn.curr)
|
||
|
|
goto done;
|
||
|
|
|
||
|
|
mask = priv->opfn.requested & ~priv->opfn.completed;
|
||
|
|
capcode = ilog2(mask & ~(mask - 1)) + 1;
|
||
|
|
if (capcode >= STL_VERBS_EXTD_MAX) {
|
||
|
|
priv->opfn.completed |= OPFN_CODE(capcode);
|
||
|
|
goto done;
|
||
|
|
}
|
||
|
|
|
||
|
|
extd = &hfi1_opfn_handlers[capcode];
|
||
|
|
if (!extd || !extd->request || !extd->request(qp, &data)) {
|
||
|
|
/*
|
||
|
|
* Either there is no handler for this capability or the request
|
||
|
|
* packet could not be generated. Either way, mark it as done so
|
||
|
|
* we don't keep attempting to complete it.
|
||
|
|
*/
|
||
|
|
priv->opfn.completed |= OPFN_CODE(capcode);
|
||
|
|
goto done;
|
||
|
|
}
|
||
|
|
|
||
|
|
trace_hfi1_opfn_data_conn_request(qp, capcode, data);
|
||
|
|
data = (data & ~0xf) | capcode;
|
||
|
|
|
||
|
|
memset(&wr, 0, sizeof(wr));
|
||
|
|
wr.wr.opcode = IB_WR_OPFN;
|
||
|
|
wr.remote_addr = HFI1_VERBS_E_ATOMIC_VADDR;
|
||
|
|
wr.compare_add = data;
|
||
|
|
|
||
|
|
priv->opfn.curr = capcode; /* A new request is now in progress */
|
||
|
|
/* Drop opfn.lock before calling ib_post_send() */
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
|
||
|
|
ret = ib_post_send(&qp->ibqp, &wr.wr, NULL);
|
||
|
|
if (ret)
|
||
|
|
goto err;
|
||
|
|
trace_hfi1_opfn_state_conn_request(qp);
|
||
|
|
return;
|
||
|
|
err:
|
||
|
|
trace_hfi1_msg_opfn_conn_request(qp, "ib_ost_send failed: ret = ",
|
||
|
|
(u64)ret);
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
/*
|
||
|
|
* In case of an unexpected error return from ib_post_send
|
||
|
|
* clear opfn.curr and reschedule to try again
|
||
|
|
*/
|
||
|
|
priv->opfn.curr = STL_VERBS_EXTD_NONE;
|
||
|
|
opfn_schedule_conn_request(qp);
|
||
|
|
done:
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_send_conn_request(struct work_struct *work)
|
||
|
|
{
|
||
|
|
struct hfi1_opfn_data *od;
|
||
|
|
struct hfi1_qp_priv *qpriv;
|
||
|
|
|
||
|
|
od = container_of(work, struct hfi1_opfn_data, opfn_work);
|
||
|
|
qpriv = container_of(od, struct hfi1_qp_priv, opfn);
|
||
|
|
|
||
|
|
opfn_conn_request(qpriv->owner);
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* When QP s_lock is held in the caller, the OPFN request must be scheduled
|
||
|
|
* to a different workqueue to avoid double locking QP s_lock in call to
|
||
|
|
* ib_post_send in opfn_conn_request
|
||
|
|
*/
|
||
|
|
static void opfn_schedule_conn_request(struct rvt_qp *qp)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
|
||
|
|
trace_hfi1_opfn_state_sched_conn_request(qp);
|
||
|
|
queue_work(opfn_wq, &priv->opfn.opfn_work);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e,
|
||
|
|
struct ib_atomic_eth *ateth)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
u64 data = be64_to_cpu(ateth->compare_data);
|
||
|
|
struct hfi1_opfn_type *extd;
|
||
|
|
u8 capcode;
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
trace_hfi1_opfn_state_conn_response(qp);
|
||
|
|
capcode = data & 0xf;
|
||
|
|
trace_hfi1_opfn_data_conn_response(qp, capcode, data);
|
||
|
|
if (!capcode || capcode >= STL_VERBS_EXTD_MAX)
|
||
|
|
return;
|
||
|
|
|
||
|
|
extd = &hfi1_opfn_handlers[capcode];
|
||
|
|
|
||
|
|
if (!extd || !extd->response) {
|
||
|
|
e->atomic_data = capcode;
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
if (priv->opfn.completed & OPFN_CODE(capcode)) {
|
||
|
|
/*
|
||
|
|
* We are receiving a request for a feature that has already
|
||
|
|
* been negotiated. This may mean that the other side has reset
|
||
|
|
*/
|
||
|
|
priv->opfn.completed &= ~OPFN_CODE(capcode);
|
||
|
|
if (extd->error)
|
||
|
|
extd->error(qp);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (extd->response(qp, &data))
|
||
|
|
priv->opfn.completed |= OPFN_CODE(capcode);
|
||
|
|
e->atomic_data = (data & ~0xf) | capcode;
|
||
|
|
trace_hfi1_opfn_state_conn_response(qp);
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_conn_reply(struct rvt_qp *qp, u64 data)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
struct hfi1_opfn_type *extd;
|
||
|
|
u8 capcode;
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
trace_hfi1_opfn_state_conn_reply(qp);
|
||
|
|
capcode = data & 0xf;
|
||
|
|
trace_hfi1_opfn_data_conn_reply(qp, capcode, data);
|
||
|
|
if (!capcode || capcode >= STL_VERBS_EXTD_MAX)
|
||
|
|
return;
|
||
|
|
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
/*
|
||
|
|
* Either there is no previous request or the reply is not for the
|
||
|
|
* current request
|
||
|
|
*/
|
||
|
|
if (!priv->opfn.curr || capcode != priv->opfn.curr)
|
||
|
|
goto done;
|
||
|
|
|
||
|
|
extd = &hfi1_opfn_handlers[capcode];
|
||
|
|
|
||
|
|
if (!extd || !extd->reply)
|
||
|
|
goto clear;
|
||
|
|
|
||
|
|
if (extd->reply(qp, data))
|
||
|
|
priv->opfn.completed |= OPFN_CODE(capcode);
|
||
|
|
clear:
|
||
|
|
/*
|
||
|
|
* Clear opfn.curr to indicate that the previous request is no longer in
|
||
|
|
* progress
|
||
|
|
*/
|
||
|
|
priv->opfn.curr = STL_VERBS_EXTD_NONE;
|
||
|
|
trace_hfi1_opfn_state_conn_reply(qp);
|
||
|
|
done:
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_conn_error(struct rvt_qp *qp)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
struct hfi1_opfn_type *extd = NULL;
|
||
|
|
unsigned long flags;
|
||
|
|
u16 capcode;
|
||
|
|
|
||
|
|
trace_hfi1_opfn_state_conn_error(qp);
|
||
|
|
trace_hfi1_msg_opfn_conn_error(qp, "error. qp state ", (u64)qp->state);
|
||
|
|
/*
|
||
|
|
* The QP has gone into the Error state. We have to invalidate all
|
||
|
|
* negotiated feature, including the one in progress (if any). The RC
|
||
|
|
* QP handling will clean the WQE for the connection request.
|
||
|
|
*/
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
while (priv->opfn.completed) {
|
||
|
|
capcode = priv->opfn.completed & ~(priv->opfn.completed - 1);
|
||
|
|
extd = &hfi1_opfn_handlers[ilog2(capcode) + 1];
|
||
|
|
if (extd->error)
|
||
|
|
extd->error(qp);
|
||
|
|
priv->opfn.completed &= ~OPFN_CODE(capcode);
|
||
|
|
}
|
||
|
|
priv->opfn.extended = 0;
|
||
|
|
priv->opfn.requested = 0;
|
||
|
|
priv->opfn.curr = STL_VERBS_EXTD_NONE;
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask)
|
||
|
|
{
|
||
|
|
struct ib_qp *ibqp = &qp->ibqp;
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
unsigned long flags;
|
||
|
|
|
||
|
|
if (attr_mask & IB_QP_RETRY_CNT)
|
||
|
|
priv->s_retry = attr->retry_cnt;
|
||
|
|
|
||
|
|
spin_lock_irqsave(&priv->opfn.lock, flags);
|
||
|
|
if (ibqp->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) {
|
||
|
|
struct tid_rdma_params *local = &priv->tid_rdma.local;
|
||
|
|
|
||
|
|
if (attr_mask & IB_QP_TIMEOUT)
|
||
|
|
priv->tid_retry_timeout_jiffies = qp->timeout_jiffies;
|
||
|
|
if (qp->pmtu == enum_to_mtu(OPA_MTU_4096) ||
|
||
|
|
qp->pmtu == enum_to_mtu(OPA_MTU_8192)) {
|
||
|
|
tid_rdma_opfn_init(qp, local);
|
||
|
|
/*
|
||
|
|
* We only want to set the OPFN requested bit when the
|
||
|
|
* QP transitions to RTS.
|
||
|
|
*/
|
||
|
|
if (attr_mask & IB_QP_STATE &&
|
||
|
|
attr->qp_state == IB_QPS_RTS) {
|
||
|
|
priv->opfn.requested |= OPFN_MASK(TID_RDMA);
|
||
|
|
/*
|
||
|
|
* If the QP is transitioning to RTS and the
|
||
|
|
* opfn.completed for TID RDMA has already been
|
||
|
|
* set, the QP is being moved *back* into RTS.
|
||
|
|
* We can now renegotiate the TID RDMA
|
||
|
|
* parameters.
|
||
|
|
*/
|
||
|
|
if (priv->opfn.completed &
|
||
|
|
OPFN_MASK(TID_RDMA)) {
|
||
|
|
priv->opfn.completed &=
|
||
|
|
~OPFN_MASK(TID_RDMA);
|
||
|
|
/*
|
||
|
|
* Since the opfn.completed bit was
|
||
|
|
* already set, it is safe to assume
|
||
|
|
* that the opfn.extended is also set.
|
||
|
|
*/
|
||
|
|
opfn_schedule_conn_request(qp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
memset(local, 0, sizeof(*local));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
spin_unlock_irqrestore(&priv->opfn.lock, flags);
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1)
|
||
|
|
{
|
||
|
|
struct hfi1_qp_priv *priv = qp->priv;
|
||
|
|
|
||
|
|
if (!priv->opfn.extended && hfi1_opfn_extended(bth1) &&
|
||
|
|
HFI1_CAP_IS_KSET(OPFN)) {
|
||
|
|
priv->opfn.extended = 1;
|
||
|
|
if (qp->state == IB_QPS_RTS)
|
||
|
|
opfn_conn_request(qp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int opfn_init(void)
|
||
|
|
{
|
||
|
|
opfn_wq = alloc_workqueue("hfi_opfn",
|
||
|
|
WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |
|
||
|
|
WQ_MEM_RECLAIM,
|
||
|
|
HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES);
|
||
|
|
if (!opfn_wq)
|
||
|
|
return -ENOMEM;
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
void opfn_exit(void)
|
||
|
|
{
|
||
|
|
if (opfn_wq) {
|
||
|
|
destroy_workqueue(opfn_wq);
|
||
|
|
opfn_wq = NULL;
|
||
|
|
}
|
||
|
|
}
|