376 lines
13 KiB
C
376 lines
13 KiB
C
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
|
|
/* Copyright 2021 Marvell. All rights reserved. */
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/module.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/list.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/types.h>
|
|
#include <asm/byteorder.h>
|
|
#include <linux/qed/common_hsi.h>
|
|
#include <linux/qed/storage_common.h>
|
|
#include <linux/qed/nvmetcp_common.h>
|
|
#include <linux/qed/qed_nvmetcp_if.h>
|
|
#include "qed_nvmetcp_fw_funcs.h"
|
|
|
|
#define NVMETCP_NUM_SGES_IN_CACHE 0x4
|
|
|
|
bool nvmetcp_is_slow_sgl(u16 num_sges, bool small_mid_sge)
|
|
{
|
|
return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge);
|
|
}
|
|
|
|
void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params,
|
|
struct scsi_cached_sges *ctx_data_desc,
|
|
struct storage_sgl_task_params *sgl_params)
|
|
{
|
|
u8 num_sges_to_init = (u8)(sgl_params->num_sges > NVMETCP_NUM_SGES_IN_CACHE ?
|
|
NVMETCP_NUM_SGES_IN_CACHE : sgl_params->num_sges);
|
|
u8 sge_index;
|
|
|
|
/* sgl params */
|
|
ctx_sgl_params->sgl_addr.lo = cpu_to_le32(sgl_params->sgl_phys_addr.lo);
|
|
ctx_sgl_params->sgl_addr.hi = cpu_to_le32(sgl_params->sgl_phys_addr.hi);
|
|
ctx_sgl_params->sgl_total_length = cpu_to_le32(sgl_params->total_buffer_size);
|
|
ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_params->num_sges);
|
|
|
|
for (sge_index = 0; sge_index < num_sges_to_init; sge_index++) {
|
|
ctx_data_desc->sge[sge_index].sge_addr.lo =
|
|
cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.lo);
|
|
ctx_data_desc->sge[sge_index].sge_addr.hi =
|
|
cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.hi);
|
|
ctx_data_desc->sge[sge_index].sge_len =
|
|
cpu_to_le32(sgl_params->sgl[sge_index].sge_len);
|
|
}
|
|
}
|
|
|
|
static inline u32 calc_rw_task_size(struct nvmetcp_task_params *task_params,
|
|
enum nvmetcp_task_type task_type)
|
|
{
|
|
u32 io_size;
|
|
|
|
if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE)
|
|
io_size = task_params->tx_io_size;
|
|
else
|
|
io_size = task_params->rx_io_size;
|
|
|
|
if (unlikely(!io_size))
|
|
return 0;
|
|
|
|
return io_size;
|
|
}
|
|
|
|
static inline void init_sqe(struct nvmetcp_task_params *task_params,
|
|
struct storage_sgl_task_params *sgl_task_params,
|
|
enum nvmetcp_task_type task_type)
|
|
{
|
|
if (!task_params->sqe)
|
|
return;
|
|
|
|
memset(task_params->sqe, 0, sizeof(*task_params->sqe));
|
|
task_params->sqe->task_id = cpu_to_le16(task_params->itid);
|
|
|
|
switch (task_type) {
|
|
case NVMETCP_TASK_TYPE_HOST_WRITE: {
|
|
u32 buf_size = 0;
|
|
u32 num_sges = 0;
|
|
|
|
SET_FIELD(task_params->sqe->contlen_cdbsize,
|
|
NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
|
|
NVMETCP_WQE_TYPE_NORMAL);
|
|
if (task_params->tx_io_size) {
|
|
if (task_params->send_write_incapsule)
|
|
buf_size = calc_rw_task_size(task_params, task_type);
|
|
|
|
if (nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
|
|
sgl_task_params->small_mid_sge))
|
|
num_sges = NVMETCP_WQE_NUM_SGES_SLOWIO;
|
|
else
|
|
num_sges = min((u16)sgl_task_params->num_sges,
|
|
(u16)SCSI_NUM_SGES_SLOW_SGL_THR);
|
|
}
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, num_sges);
|
|
SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, buf_size);
|
|
} break;
|
|
|
|
case NVMETCP_TASK_TYPE_HOST_READ: {
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
|
|
NVMETCP_WQE_TYPE_NORMAL);
|
|
SET_FIELD(task_params->sqe->contlen_cdbsize,
|
|
NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
|
|
} break;
|
|
|
|
case NVMETCP_TASK_TYPE_INIT_CONN_REQUEST: {
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
|
|
NVMETCP_WQE_TYPE_MIDDLE_PATH);
|
|
|
|
if (task_params->tx_io_size) {
|
|
SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN,
|
|
task_params->tx_io_size);
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES,
|
|
min((u16)sgl_task_params->num_sges,
|
|
(u16)SCSI_NUM_SGES_SLOW_SGL_THR));
|
|
}
|
|
} break;
|
|
|
|
case NVMETCP_TASK_TYPE_CLEANUP:
|
|
SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
|
|
NVMETCP_WQE_TYPE_TASK_CLEANUP);
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* The following function initializes of NVMeTCP task params */
|
|
static inline void
|
|
init_nvmetcp_task_params(struct e5_nvmetcp_task_context *context,
|
|
struct nvmetcp_task_params *task_params,
|
|
enum nvmetcp_task_type task_type)
|
|
{
|
|
context->ystorm_st_context.state.cccid = task_params->host_cccid;
|
|
SET_FIELD(context->ustorm_st_context.error_flags, USTORM_NVMETCP_TASK_ST_CTX_NVME_TCP, 1);
|
|
context->ustorm_st_context.nvme_tcp_opaque_lo = cpu_to_le32(task_params->opq.lo);
|
|
context->ustorm_st_context.nvme_tcp_opaque_hi = cpu_to_le32(task_params->opq.hi);
|
|
}
|
|
|
|
/* The following function initializes default values to all tasks */
|
|
static inline void
|
|
init_default_nvmetcp_task(struct nvmetcp_task_params *task_params,
|
|
void *pdu_header, void *nvme_cmd,
|
|
enum nvmetcp_task_type task_type)
|
|
{
|
|
struct e5_nvmetcp_task_context *context = task_params->context;
|
|
const u8 val_byte = context->mstorm_ag_context.cdu_validation;
|
|
u8 dw_index;
|
|
|
|
memset(context, 0, sizeof(*context));
|
|
init_nvmetcp_task_params(context, task_params,
|
|
(enum nvmetcp_task_type)task_type);
|
|
|
|
/* Swapping requirements used below, will be removed in future FW versions */
|
|
if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE ||
|
|
task_type == NVMETCP_TASK_TYPE_HOST_READ) {
|
|
for (dw_index = 0;
|
|
dw_index < QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
|
|
dw_index++)
|
|
context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
|
|
cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
|
|
|
|
for (dw_index = QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
|
|
dw_index < QED_NVMETCP_CMD_HDR_SIZE / sizeof(u32);
|
|
dw_index++)
|
|
context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
|
|
cpu_to_le32(__swab32(((u32 *)nvme_cmd)[dw_index - 2]));
|
|
} else {
|
|
for (dw_index = 0;
|
|
dw_index < QED_NVMETCP_NON_IO_HDR_SIZE / sizeof(u32);
|
|
dw_index++)
|
|
context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
|
|
cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
|
|
}
|
|
|
|
/* M-Storm Context: */
|
|
context->mstorm_ag_context.cdu_validation = val_byte;
|
|
context->mstorm_st_context.task_type = (u8)(task_type);
|
|
context->mstorm_ag_context.task_cid = cpu_to_le16(task_params->conn_icid);
|
|
|
|
/* Ustorm Context: */
|
|
SET_FIELD(context->ustorm_ag_context.flags1, E5_USTORM_NVMETCP_TASK_AG_CTX_R2T2RECV, 1);
|
|
context->ustorm_st_context.task_type = (u8)(task_type);
|
|
context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number;
|
|
context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid);
|
|
}
|
|
|
|
/* The following function initializes the U-Storm Task Contexts */
|
|
static inline void
|
|
init_ustorm_task_contexts(struct ustorm_nvmetcp_task_st_ctx *ustorm_st_context,
|
|
struct e5_ustorm_nvmetcp_task_ag_ctx *ustorm_ag_context,
|
|
u32 remaining_recv_len,
|
|
u32 expected_data_transfer_len, u8 num_sges,
|
|
bool tx_dif_conn_err_en)
|
|
{
|
|
/* Remaining data to be received in bytes. Used in validations*/
|
|
ustorm_st_context->rem_rcv_len = cpu_to_le32(remaining_recv_len);
|
|
ustorm_ag_context->exp_data_acked = cpu_to_le32(expected_data_transfer_len);
|
|
ustorm_st_context->exp_data_transfer_len = cpu_to_le32(expected_data_transfer_len);
|
|
SET_FIELD(ustorm_st_context->reg1_map, REG1_NUM_SGES, num_sges);
|
|
SET_FIELD(ustorm_ag_context->flags2, E5_USTORM_NVMETCP_TASK_AG_CTX_DIF_ERROR_CF_EN,
|
|
tx_dif_conn_err_en ? 1 : 0);
|
|
}
|
|
|
|
/* The following function initializes Local Completion Contexts: */
|
|
static inline void
|
|
set_local_completion_context(struct e5_nvmetcp_task_context *context)
|
|
{
|
|
SET_FIELD(context->ystorm_st_context.state.flags,
|
|
YSTORM_NVMETCP_TASK_STATE_LOCAL_COMP, 1);
|
|
SET_FIELD(context->ustorm_st_context.flags,
|
|
USTORM_NVMETCP_TASK_ST_CTX_LOCAL_COMP, 1);
|
|
}
|
|
|
|
/* Common Fastpath task init function: */
|
|
static inline void
|
|
init_rw_nvmetcp_task(struct nvmetcp_task_params *task_params,
|
|
enum nvmetcp_task_type task_type,
|
|
void *pdu_header, void *nvme_cmd,
|
|
struct storage_sgl_task_params *sgl_task_params)
|
|
{
|
|
struct e5_nvmetcp_task_context *context = task_params->context;
|
|
u32 task_size = calc_rw_task_size(task_params, task_type);
|
|
bool slow_io = false;
|
|
u8 num_sges = 0;
|
|
|
|
init_default_nvmetcp_task(task_params, pdu_header, nvme_cmd, task_type);
|
|
|
|
/* Tx/Rx: */
|
|
if (task_params->tx_io_size) {
|
|
/* if data to transmit: */
|
|
init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
|
|
&context->ystorm_st_context.state.data_desc,
|
|
sgl_task_params);
|
|
slow_io = nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
|
|
sgl_task_params->small_mid_sge);
|
|
num_sges =
|
|
(u8)(!slow_io ? min((u32)sgl_task_params->num_sges,
|
|
(u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
|
|
NVMETCP_WQE_NUM_SGES_SLOWIO);
|
|
if (slow_io) {
|
|
SET_FIELD(context->ystorm_st_context.state.flags,
|
|
YSTORM_NVMETCP_TASK_STATE_SLOW_IO, 1);
|
|
}
|
|
} else if (task_params->rx_io_size) {
|
|
/* if data to receive: */
|
|
init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
|
|
&context->mstorm_st_context.data_desc,
|
|
sgl_task_params);
|
|
num_sges =
|
|
(u8)(!nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
|
|
sgl_task_params->small_mid_sge) ?
|
|
min((u32)sgl_task_params->num_sges,
|
|
(u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
|
|
NVMETCP_WQE_NUM_SGES_SLOWIO);
|
|
context->mstorm_st_context.rem_task_size = cpu_to_le32(task_size);
|
|
}
|
|
|
|
/* Ustorm context: */
|
|
init_ustorm_task_contexts(&context->ustorm_st_context,
|
|
&context->ustorm_ag_context,
|
|
/* Remaining Receive length is the Task Size */
|
|
task_size,
|
|
/* The size of the transmitted task */
|
|
task_size,
|
|
/* num_sges */
|
|
num_sges,
|
|
false);
|
|
|
|
/* Set exp_data_acked */
|
|
if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) {
|
|
if (task_params->send_write_incapsule)
|
|
context->ustorm_ag_context.exp_data_acked = task_size;
|
|
else
|
|
context->ustorm_ag_context.exp_data_acked = 0;
|
|
} else if (task_type == NVMETCP_TASK_TYPE_HOST_READ) {
|
|
context->ustorm_ag_context.exp_data_acked = 0;
|
|
}
|
|
|
|
context->ustorm_ag_context.exp_cont_len = 0;
|
|
init_sqe(task_params, sgl_task_params, task_type);
|
|
}
|
|
|
|
static void
|
|
init_common_initiator_read_task(struct nvmetcp_task_params *task_params,
|
|
struct nvme_tcp_cmd_pdu *cmd_pdu_header,
|
|
struct nvme_command *nvme_cmd,
|
|
struct storage_sgl_task_params *sgl_task_params)
|
|
{
|
|
init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_READ,
|
|
cmd_pdu_header, nvme_cmd, sgl_task_params);
|
|
}
|
|
|
|
void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params,
|
|
struct nvme_tcp_cmd_pdu *cmd_pdu_header,
|
|
struct nvme_command *nvme_cmd,
|
|
struct storage_sgl_task_params *sgl_task_params)
|
|
{
|
|
init_common_initiator_read_task(task_params, (void *)cmd_pdu_header,
|
|
(void *)nvme_cmd, sgl_task_params);
|
|
}
|
|
|
|
static void
|
|
init_common_initiator_write_task(struct nvmetcp_task_params *task_params,
|
|
struct nvme_tcp_cmd_pdu *cmd_pdu_header,
|
|
struct nvme_command *nvme_cmd,
|
|
struct storage_sgl_task_params *sgl_task_params)
|
|
{
|
|
init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_WRITE,
|
|
cmd_pdu_header, nvme_cmd, sgl_task_params);
|
|
}
|
|
|
|
void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params,
|
|
struct nvme_tcp_cmd_pdu *cmd_pdu_header,
|
|
struct nvme_command *nvme_cmd,
|
|
struct storage_sgl_task_params *sgl_task_params)
|
|
{
|
|
init_common_initiator_write_task(task_params, (void *)cmd_pdu_header,
|
|
(void *)nvme_cmd, sgl_task_params);
|
|
}
|
|
|
|
static void
|
|
init_common_login_request_task(struct nvmetcp_task_params *task_params,
|
|
void *login_req_pdu_header,
|
|
struct storage_sgl_task_params *tx_sgl_task_params,
|
|
struct storage_sgl_task_params *rx_sgl_task_params)
|
|
{
|
|
struct e5_nvmetcp_task_context *context = task_params->context;
|
|
|
|
init_default_nvmetcp_task(task_params, (void *)login_req_pdu_header, NULL,
|
|
NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
|
|
|
|
/* Ustorm Context: */
|
|
init_ustorm_task_contexts(&context->ustorm_st_context,
|
|
&context->ustorm_ag_context,
|
|
|
|
/* Remaining Receive length is the Task Size */
|
|
task_params->rx_io_size ?
|
|
rx_sgl_task_params->total_buffer_size : 0,
|
|
|
|
/* The size of the transmitted task */
|
|
task_params->tx_io_size ?
|
|
tx_sgl_task_params->total_buffer_size : 0,
|
|
0, /* num_sges */
|
|
0); /* tx_dif_conn_err_en */
|
|
|
|
/* SGL context: */
|
|
if (task_params->tx_io_size)
|
|
init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
|
|
&context->ystorm_st_context.state.data_desc,
|
|
tx_sgl_task_params);
|
|
if (task_params->rx_io_size)
|
|
init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
|
|
&context->mstorm_st_context.data_desc,
|
|
rx_sgl_task_params);
|
|
|
|
context->mstorm_st_context.rem_task_size =
|
|
cpu_to_le32(task_params->rx_io_size ?
|
|
rx_sgl_task_params->total_buffer_size : 0);
|
|
init_sqe(task_params, tx_sgl_task_params, NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
|
|
}
|
|
|
|
/* The following function initializes Login task in Host mode: */
|
|
void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params,
|
|
struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr,
|
|
struct storage_sgl_task_params *tx_sgl_task_params,
|
|
struct storage_sgl_task_params *rx_sgl_task_params)
|
|
{
|
|
init_common_login_request_task(task_params, init_conn_req_pdu_hdr,
|
|
tx_sgl_task_params, rx_sgl_task_params);
|
|
}
|
|
|
|
void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params)
|
|
{
|
|
init_sqe(task_params, NULL, NVMETCP_TASK_TYPE_CLEANUP);
|
|
}
|