350 lines
8.9 KiB
C
350 lines
8.9 KiB
C
|
// SPDX-License-Identifier: GPL-2.0 OR MIT
|
||
|
|
/*
|
||
|
|
* Copyright 2018-2022 Advanced Micro Devices, Inc.
|
||
|
|
*
|
||
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||
|
|
* copy of this software and associated documentation files (the "Software"),
|
||
|
|
* to deal in the Software without restriction, including without limitation
|
||
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||
|
|
* Software is furnished to do so, subject to the following conditions:
|
||
|
|
*
|
||
|
|
* The above copyright notice and this permission notice shall be included in
|
||
|
|
* all copies or substantial portions of the Software.
|
||
|
|
*
|
||
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||
|
|
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||
|
|
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||
|
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <linux/kconfig.h>
|
||
|
|
|
||
|
|
#if IS_REACHABLE(CONFIG_AMD_IOMMU_V2)
|
||
|
|
|
||
|
|
#include <linux/printk.h>
|
||
|
|
#include <linux/device.h>
|
||
|
|
#include <linux/slab.h>
|
||
|
|
#include <linux/pci.h>
|
||
|
|
#include <linux/amd-iommu.h>
|
||
|
|
#include "kfd_priv.h"
|
||
|
|
#include "kfd_topology.h"
|
||
|
|
#include "kfd_iommu.h"
|
||
|
|
|
||
|
|
static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
|
||
|
|
AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
|
||
|
|
AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
|
||
|
|
|
||
|
|
/** kfd_iommu_check_device - Check whether IOMMU is available for device
|
||
|
|
*/
|
||
|
|
int kfd_iommu_check_device(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
struct amd_iommu_device_info iommu_info;
|
||
|
|
int err;
|
||
|
|
|
||
|
|
if (!kfd->use_iommu_v2)
|
||
|
|
return -ENODEV;
|
||
|
|
|
||
|
|
iommu_info.flags = 0;
|
||
|
|
err = amd_iommu_device_info(kfd->adev->pdev, &iommu_info);
|
||
|
|
if (err)
|
||
|
|
return err;
|
||
|
|
|
||
|
|
if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
|
||
|
|
return -ENODEV;
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_device_init - Initialize IOMMU for device
|
||
|
|
*/
|
||
|
|
int kfd_iommu_device_init(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
struct amd_iommu_device_info iommu_info;
|
||
|
|
unsigned int pasid_limit;
|
||
|
|
int err;
|
||
|
|
|
||
|
|
if (!kfd->use_iommu_v2)
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
iommu_info.flags = 0;
|
||
|
|
err = amd_iommu_device_info(kfd->adev->pdev, &iommu_info);
|
||
|
|
if (err < 0) {
|
||
|
|
dev_err(kfd_device,
|
||
|
|
"error getting iommu info. is the iommu enabled?\n");
|
||
|
|
return -ENODEV;
|
||
|
|
}
|
||
|
|
|
||
|
|
if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
|
||
|
|
dev_err(kfd_device,
|
||
|
|
"error required iommu flags ats %i, pri %i, pasid %i\n",
|
||
|
|
(iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
|
||
|
|
(iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
|
||
|
|
(iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
|
||
|
|
!= 0);
|
||
|
|
return -ENODEV;
|
||
|
|
}
|
||
|
|
|
||
|
|
pasid_limit = min_t(unsigned int,
|
||
|
|
(unsigned int)(1 << kfd->device_info.max_pasid_bits),
|
||
|
|
iommu_info.max_pasids);
|
||
|
|
|
||
|
|
if (!kfd_set_pasid_limit(pasid_limit)) {
|
||
|
|
dev_err(kfd_device, "error setting pasid limit\n");
|
||
|
|
return -EBUSY;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
|
||
|
|
*
|
||
|
|
* Binds the given process to the given device using its PASID. This
|
||
|
|
* enables IOMMUv2 address translation for the process on the device.
|
||
|
|
*
|
||
|
|
* This function assumes that the process mutex is held.
|
||
|
|
*/
|
||
|
|
int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
|
||
|
|
{
|
||
|
|
struct kfd_dev *dev = pdd->dev;
|
||
|
|
struct kfd_process *p = pdd->process;
|
||
|
|
int err;
|
||
|
|
|
||
|
|
if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
|
||
|
|
pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
|
||
|
|
return -EINVAL;
|
||
|
|
}
|
||
|
|
|
||
|
|
err = amd_iommu_bind_pasid(dev->adev->pdev, p->pasid, p->lead_thread);
|
||
|
|
if (!err)
|
||
|
|
pdd->bound = PDD_BOUND;
|
||
|
|
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_unbind_process - Unbind process from all devices
|
||
|
|
*
|
||
|
|
* This removes all IOMMU device bindings of the process. To be used
|
||
|
|
* before process termination.
|
||
|
|
*/
|
||
|
|
void kfd_iommu_unbind_process(struct kfd_process *p)
|
||
|
|
{
|
||
|
|
int i;
|
||
|
|
|
||
|
|
for (i = 0; i < p->n_pdds; i++)
|
||
|
|
if (p->pdds[i]->bound == PDD_BOUND)
|
||
|
|
amd_iommu_unbind_pasid(p->pdds[i]->dev->adev->pdev,
|
||
|
|
p->pasid);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* Callback for process shutdown invoked by the IOMMU driver */
|
||
|
|
static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
|
||
|
|
{
|
||
|
|
struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
|
||
|
|
struct kfd_process *p;
|
||
|
|
struct kfd_process_device *pdd;
|
||
|
|
|
||
|
|
if (!dev)
|
||
|
|
return;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Look for the process that matches the pasid. If there is no such
|
||
|
|
* process, we either released it in amdkfd's own notifier, or there
|
||
|
|
* is a bug. Unfortunately, there is no way to tell...
|
||
|
|
*/
|
||
|
|
p = kfd_lookup_process_by_pasid(pasid);
|
||
|
|
if (!p)
|
||
|
|
return;
|
||
|
|
|
||
|
|
pr_debug("Unbinding process 0x%x from IOMMU\n", pasid);
|
||
|
|
|
||
|
|
mutex_lock(&p->mutex);
|
||
|
|
|
||
|
|
pdd = kfd_get_process_device_data(dev, p);
|
||
|
|
if (pdd)
|
||
|
|
/* For GPU relying on IOMMU, we need to dequeue here
|
||
|
|
* when PASID is still bound.
|
||
|
|
*/
|
||
|
|
kfd_process_dequeue_from_device(pdd);
|
||
|
|
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
|
||
|
|
kfd_unref_process(p);
|
||
|
|
}
|
||
|
|
|
||
|
|
/* This function called by IOMMU driver on PPR failure */
|
||
|
|
static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
|
||
|
|
unsigned long address, u16 flags)
|
||
|
|
{
|
||
|
|
struct kfd_dev *dev;
|
||
|
|
|
||
|
|
dev_warn_ratelimited(kfd_device,
|
||
|
|
"Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
|
||
|
|
pdev->bus->number,
|
||
|
|
PCI_SLOT(pdev->devfn),
|
||
|
|
PCI_FUNC(pdev->devfn),
|
||
|
|
pasid,
|
||
|
|
address,
|
||
|
|
flags);
|
||
|
|
|
||
|
|
dev = kfd_device_by_pci_dev(pdev);
|
||
|
|
if (!WARN_ON(!dev))
|
||
|
|
kfd_signal_iommu_event(dev, pasid, address,
|
||
|
|
flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
|
||
|
|
|
||
|
|
return AMD_IOMMU_INV_PRI_RSP_INVALID;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Bind processes do the device that have been temporarily unbound
|
||
|
|
* (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
|
||
|
|
*/
|
||
|
|
static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
struct kfd_process_device *pdd;
|
||
|
|
struct kfd_process *p;
|
||
|
|
unsigned int temp;
|
||
|
|
int err = 0;
|
||
|
|
|
||
|
|
int idx = srcu_read_lock(&kfd_processes_srcu);
|
||
|
|
|
||
|
|
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
|
||
|
|
mutex_lock(&p->mutex);
|
||
|
|
pdd = kfd_get_process_device_data(kfd, p);
|
||
|
|
|
||
|
|
if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
err = amd_iommu_bind_pasid(kfd->adev->pdev, p->pasid,
|
||
|
|
p->lead_thread);
|
||
|
|
if (err < 0) {
|
||
|
|
pr_err("Unexpected pasid 0x%x binding failure\n",
|
||
|
|
p->pasid);
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
pdd->bound = PDD_BOUND;
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
}
|
||
|
|
|
||
|
|
srcu_read_unlock(&kfd_processes_srcu, idx);
|
||
|
|
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Mark currently bound processes as PDD_BOUND_SUSPENDED. These
|
||
|
|
* processes will be restored to PDD_BOUND state in
|
||
|
|
* kfd_bind_processes_to_device.
|
||
|
|
*/
|
||
|
|
static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
struct kfd_process_device *pdd;
|
||
|
|
struct kfd_process *p;
|
||
|
|
unsigned int temp;
|
||
|
|
|
||
|
|
int idx = srcu_read_lock(&kfd_processes_srcu);
|
||
|
|
|
||
|
|
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
|
||
|
|
mutex_lock(&p->mutex);
|
||
|
|
pdd = kfd_get_process_device_data(kfd, p);
|
||
|
|
|
||
|
|
if (WARN_ON(!pdd)) {
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (pdd->bound == PDD_BOUND)
|
||
|
|
pdd->bound = PDD_BOUND_SUSPENDED;
|
||
|
|
mutex_unlock(&p->mutex);
|
||
|
|
}
|
||
|
|
|
||
|
|
srcu_read_unlock(&kfd_processes_srcu, idx);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_suspend - Prepare IOMMU for suspend
|
||
|
|
*
|
||
|
|
* This unbinds processes from the device and disables the IOMMU for
|
||
|
|
* the device.
|
||
|
|
*/
|
||
|
|
void kfd_iommu_suspend(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
if (!kfd->use_iommu_v2)
|
||
|
|
return;
|
||
|
|
|
||
|
|
kfd_unbind_processes_from_device(kfd);
|
||
|
|
|
||
|
|
amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL);
|
||
|
|
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL);
|
||
|
|
amd_iommu_free_device(kfd->adev->pdev);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_resume - Restore IOMMU after resume
|
||
|
|
*
|
||
|
|
* This reinitializes the IOMMU for the device and re-binds previously
|
||
|
|
* suspended processes to the device.
|
||
|
|
*/
|
||
|
|
int kfd_iommu_resume(struct kfd_dev *kfd)
|
||
|
|
{
|
||
|
|
unsigned int pasid_limit;
|
||
|
|
int err;
|
||
|
|
|
||
|
|
if (!kfd->use_iommu_v2)
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
pasid_limit = kfd_get_pasid_limit();
|
||
|
|
|
||
|
|
err = amd_iommu_init_device(kfd->adev->pdev, pasid_limit);
|
||
|
|
if (err)
|
||
|
|
return -ENXIO;
|
||
|
|
|
||
|
|
amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev,
|
||
|
|
iommu_pasid_shutdown_callback);
|
||
|
|
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev,
|
||
|
|
iommu_invalid_ppr_cb);
|
||
|
|
|
||
|
|
err = kfd_bind_processes_to_device(kfd);
|
||
|
|
if (err) {
|
||
|
|
amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL);
|
||
|
|
amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL);
|
||
|
|
amd_iommu_free_device(kfd->adev->pdev);
|
||
|
|
return err;
|
||
|
|
}
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
|
||
|
|
*/
|
||
|
|
int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
|
||
|
|
{
|
||
|
|
struct kfd_perf_properties *props;
|
||
|
|
|
||
|
|
if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
if (!amd_iommu_pc_supported())
|
||
|
|
return 0;
|
||
|
|
|
||
|
|
props = kfd_alloc_struct(props);
|
||
|
|
if (!props)
|
||
|
|
return -ENOMEM;
|
||
|
|
strcpy(props->block_name, "iommu");
|
||
|
|
props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
|
||
|
|
amd_iommu_pc_get_max_counters(0); /* assume one iommu */
|
||
|
|
list_add_tail(&props->list, &kdev->perf_props);
|
||
|
|
|
||
|
|
return 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
#endif
|