401 lines
12 KiB
C
401 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
*
|
|
*/
|
|
|
|
/* Support for NVIDIA specific attributes. */
|
|
|
|
#include <linux/topology.h>
|
|
|
|
#include "nvidia_cspmu.h"
|
|
|
|
#define NV_PCIE_PORT_COUNT 10ULL
|
|
#define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
|
|
|
|
#define NV_NVL_C2C_PORT_COUNT 2ULL
|
|
#define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
|
|
|
|
#define NV_CNVL_PORT_COUNT 4ULL
|
|
#define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
|
|
|
|
#define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
|
|
|
|
#define NV_PRODID_MASK GENMASK(31, 0)
|
|
|
|
#define NV_FORMAT_NAME_GENERIC 0
|
|
|
|
#define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
|
|
|
|
#define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
|
|
ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
|
|
|
|
#define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
|
|
NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
|
|
NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
|
|
NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
|
|
NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
|
|
|
|
struct nv_cspmu_ctx {
|
|
const char *name;
|
|
u32 filter_mask;
|
|
u32 filter_default_val;
|
|
struct attribute **event_attr;
|
|
struct attribute **format_attr;
|
|
};
|
|
|
|
static struct attribute *scf_pmu_event_attrs[] = {
|
|
ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
|
|
ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
|
|
ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
|
|
ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp, 0x105),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp, 0x10d),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, prb_data, 0x111),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding, 0x119),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding, 0x11d),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding, 0x121),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding, 0x125),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding, 0x129),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, dl_access, 0x131),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, ev_access, 0x13d),
|
|
NV_CSPMU_EVENT_ATTR_4(socket, prb_access, 0x141),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data, 0x145),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access, 0x149),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access, 0x14d),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding, 0x151),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding, 0x155),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data, 0x159),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access, 0x15d),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access, 0x161),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding, 0x165),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding, 0x169),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp, 0x170),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_dl_access, 0x171),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding, 0x172),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding, 0x175),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp, 0x176),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_ev_access, 0x177),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding, 0x178),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding, 0x17a),
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data, 0x180),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data, 0x184),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access, 0x188),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding, 0x18c),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data, 0x190),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data, 0x194),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access, 0x198),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding, 0x19c),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
|
|
ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
|
|
ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
|
|
ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
|
|
ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp, 0x1a8),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_dl_access, 0x1a9),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding, 0x1aa),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding, 0x1ad),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp, 0x1ae),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_ev_access, 0x1af),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding, 0x1b0),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding, 0x1b2),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data, 0x1b3),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access, 0x1b7),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access, 0x1bb),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding, 0x1bf),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding, 0x1c3),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(ocu_prb_access, 0x1c7),
|
|
ARM_CSPMU_EVENT_ATTR(ocu_prb_data, 0x1c8),
|
|
ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding, 0x1c9),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
|
|
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access, 0x1cb),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data, 0x1cf),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data, 0x1d3),
|
|
NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding, 0x1d7),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
|
|
|
|
ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *mcf_pmu_event_attrs[] = {
|
|
ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
|
|
ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
|
|
ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
|
|
ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
|
|
ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
|
|
ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
|
|
ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
|
|
ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
|
|
ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
|
|
ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
|
|
ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
|
|
ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
|
|
ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
|
|
ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
|
|
ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *generic_pmu_event_attrs[] = {
|
|
ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *scf_pmu_format_attrs[] = {
|
|
ARM_CSPMU_FORMAT_EVENT_ATTR,
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *pcie_pmu_format_attrs[] = {
|
|
ARM_CSPMU_FORMAT_EVENT_ATTR,
|
|
ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
|
|
ARM_CSPMU_FORMAT_EVENT_ATTR,
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *cnvlink_pmu_format_attrs[] = {
|
|
ARM_CSPMU_FORMAT_EVENT_ATTR,
|
|
ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute *generic_pmu_format_attrs[] = {
|
|
ARM_CSPMU_FORMAT_EVENT_ATTR,
|
|
ARM_CSPMU_FORMAT_FILTER_ATTR,
|
|
NULL,
|
|
};
|
|
|
|
static struct attribute **
|
|
nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
|
|
{
|
|
const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
|
|
|
|
return ctx->event_attr;
|
|
}
|
|
|
|
static struct attribute **
|
|
nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
|
|
{
|
|
const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
|
|
|
|
return ctx->format_attr;
|
|
}
|
|
|
|
static const char *
|
|
nv_cspmu_get_name(const struct arm_cspmu *cspmu)
|
|
{
|
|
const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
|
|
|
|
return ctx->name;
|
|
}
|
|
|
|
static u32 nv_cspmu_event_filter(const struct perf_event *event)
|
|
{
|
|
const struct nv_cspmu_ctx *ctx =
|
|
to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
|
|
|
|
if (ctx->filter_mask == 0)
|
|
return ctx->filter_default_val;
|
|
|
|
return event->attr.config1 & ctx->filter_mask;
|
|
}
|
|
|
|
enum nv_cspmu_name_fmt {
|
|
NAME_FMT_GENERIC,
|
|
NAME_FMT_SOCKET
|
|
};
|
|
|
|
struct nv_cspmu_match {
|
|
u32 prodid;
|
|
u32 prodid_mask;
|
|
u64 filter_mask;
|
|
u32 filter_default_val;
|
|
const char *name_pattern;
|
|
enum nv_cspmu_name_fmt name_fmt;
|
|
struct attribute **event_attr;
|
|
struct attribute **format_attr;
|
|
};
|
|
|
|
static const struct nv_cspmu_match nv_cspmu_match[] = {
|
|
{
|
|
.prodid = 0x103,
|
|
.prodid_mask = NV_PRODID_MASK,
|
|
.filter_mask = NV_PCIE_FILTER_ID_MASK,
|
|
.filter_default_val = NV_PCIE_FILTER_ID_MASK,
|
|
.name_pattern = "nvidia_pcie_pmu_%u",
|
|
.name_fmt = NAME_FMT_SOCKET,
|
|
.event_attr = mcf_pmu_event_attrs,
|
|
.format_attr = pcie_pmu_format_attrs
|
|
},
|
|
{
|
|
.prodid = 0x104,
|
|
.prodid_mask = NV_PRODID_MASK,
|
|
.filter_mask = 0x0,
|
|
.filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
|
|
.name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
|
|
.name_fmt = NAME_FMT_SOCKET,
|
|
.event_attr = mcf_pmu_event_attrs,
|
|
.format_attr = nvlink_c2c_pmu_format_attrs
|
|
},
|
|
{
|
|
.prodid = 0x105,
|
|
.prodid_mask = NV_PRODID_MASK,
|
|
.filter_mask = 0x0,
|
|
.filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
|
|
.name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
|
|
.name_fmt = NAME_FMT_SOCKET,
|
|
.event_attr = mcf_pmu_event_attrs,
|
|
.format_attr = nvlink_c2c_pmu_format_attrs
|
|
},
|
|
{
|
|
.prodid = 0x106,
|
|
.prodid_mask = NV_PRODID_MASK,
|
|
.filter_mask = NV_CNVL_FILTER_ID_MASK,
|
|
.filter_default_val = NV_CNVL_FILTER_ID_MASK,
|
|
.name_pattern = "nvidia_cnvlink_pmu_%u",
|
|
.name_fmt = NAME_FMT_SOCKET,
|
|
.event_attr = mcf_pmu_event_attrs,
|
|
.format_attr = cnvlink_pmu_format_attrs
|
|
},
|
|
{
|
|
.prodid = 0x2CF,
|
|
.prodid_mask = NV_PRODID_MASK,
|
|
.filter_mask = 0x0,
|
|
.filter_default_val = 0x0,
|
|
.name_pattern = "nvidia_scf_pmu_%u",
|
|
.name_fmt = NAME_FMT_SOCKET,
|
|
.event_attr = scf_pmu_event_attrs,
|
|
.format_attr = scf_pmu_format_attrs
|
|
},
|
|
{
|
|
.prodid = 0,
|
|
.prodid_mask = 0,
|
|
.filter_mask = NV_GENERIC_FILTER_ID_MASK,
|
|
.filter_default_val = NV_GENERIC_FILTER_ID_MASK,
|
|
.name_pattern = "nvidia_uncore_pmu_%u",
|
|
.name_fmt = NAME_FMT_GENERIC,
|
|
.event_attr = generic_pmu_event_attrs,
|
|
.format_attr = generic_pmu_format_attrs
|
|
},
|
|
};
|
|
|
|
static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
|
|
const struct nv_cspmu_match *match)
|
|
{
|
|
char *name;
|
|
struct device *dev = cspmu->dev;
|
|
|
|
static atomic_t pmu_generic_idx = {0};
|
|
|
|
switch (match->name_fmt) {
|
|
case NAME_FMT_SOCKET: {
|
|
const int cpu = cpumask_first(&cspmu->associated_cpus);
|
|
const int socket = cpu_to_node(cpu);
|
|
|
|
name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
|
|
socket);
|
|
break;
|
|
}
|
|
case NAME_FMT_GENERIC:
|
|
name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
|
|
atomic_fetch_inc(&pmu_generic_idx));
|
|
break;
|
|
default:
|
|
name = NULL;
|
|
break;
|
|
}
|
|
|
|
return name;
|
|
}
|
|
|
|
int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
|
|
{
|
|
u32 prodid;
|
|
struct nv_cspmu_ctx *ctx;
|
|
struct device *dev = cspmu->dev;
|
|
struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
|
|
const struct nv_cspmu_match *match = nv_cspmu_match;
|
|
|
|
ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
|
|
if (!ctx)
|
|
return -ENOMEM;
|
|
|
|
prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
|
|
|
|
/* Find matching PMU. */
|
|
for (; match->prodid; match++) {
|
|
const u32 prodid_mask = match->prodid_mask;
|
|
|
|
if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
|
|
break;
|
|
}
|
|
|
|
ctx->name = nv_cspmu_format_name(cspmu, match);
|
|
ctx->filter_mask = match->filter_mask;
|
|
ctx->filter_default_val = match->filter_default_val;
|
|
ctx->event_attr = match->event_attr;
|
|
ctx->format_attr = match->format_attr;
|
|
|
|
cspmu->impl.ctx = ctx;
|
|
|
|
/* NVIDIA specific callbacks. */
|
|
impl_ops->event_filter = nv_cspmu_event_filter;
|
|
impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
|
|
impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
|
|
impl_ops->get_name = nv_cspmu_get_name;
|
|
|
|
/* Set others to NULL to use default callback. */
|
|
impl_ops->event_type = NULL;
|
|
impl_ops->event_attr_is_visible = NULL;
|
|
impl_ops->get_identifier = NULL;
|
|
impl_ops->is_cycle_counter_event = NULL;
|
|
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL_GPL(nv_cspmu_init_ops);
|
|
|
|
MODULE_LICENSE("GPL v2");
|