linux-zen-server/drivers/pci/controller/intel-nvme-remap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Intel remapped NVMe device support.
 *
 * Copyright (c) 2019 Endless Mobile, Inc.
 * Author: Daniel Drake <drake@endlessm.com>
 *
 * Some products ship by default with the SATA controller in "RAID" or
 * "Intel RST Premium With Intel Optane System Acceleration" mode. Under this
 * mode, which we refer to as "remapped NVMe" mode, any installed NVMe
 * devices disappear from the PCI bus, and instead their I/O memory becomes
 * available within the AHCI device BARs.
 *
 * This scheme is understood to be a way of avoiding usage of the standard
 * Windows NVMe driver under that OS, instead mandating usage of Intel's
 * driver instead, which has better power management, and presumably offers
 * some RAID/disk-caching solutions too.
 *
 * Here in this driver, we support the remapped NVMe mode by claiming the
 * AHCI device and creating a fake PCIe root port. On the new bus, the
 * original AHCI device is exposed with only minor tweaks. Then, fake PCI
 * devices corresponding to the remapped NVMe devices are created. The usual
 * ahci and nvme drivers are then expected to bind to these devices and
 * operate as normal.
 *
 * The PCI configuration space for the NVMe devices is completely
 * unavailable, so we fake a minimal one and hope for the best.
 *
 * Interrupts are shared between the AHCI and NVMe devices. For simplicity,
 * we only support the legacy interrupt here, although MSI support
 * could potentially be added later.
 */

#define MODULE_NAME "intel-nvme-remap"

#include <linux/ahci-remap.h>
#include <linux/irq.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/pci.h>

#define AHCI_PCI_BAR_STANDARD 5

struct nvme_remap_dev {
	struct pci_dev		*dev;		/* AHCI device */
	struct pci_bus		*bus;		/* our fake PCI bus */
	struct pci_sysdata	sysdata;
	int			irq_base;	/* our fake interrupts */

	/*
	 * When we detect an all-ones write to a BAR register, this flag
	 * is set, so that we return the BAR size on the next read (a
	 * standard PCI behaviour).
	 * This includes the assumption that an all-ones BAR write is
	 * immediately followed by a read of the same register.
	 */
	bool			bar_sizing;

	/*
	 * Resources copied from the AHCI device, to be regarded as
	 * resources on our fake bus.
	 */
	struct resource		ahci_resources[PCI_NUM_RESOURCES];

	/* Resources corresponding to the NVMe devices. */
	struct resource		remapped_dev_mem[AHCI_MAX_REMAP];

	/* Number of remapped NVMe devices found. */
	int			num_remapped_devices;
};

static inline struct nvme_remap_dev *nrdev_from_bus(struct pci_bus *bus)
{
	return container_of(bus->sysdata, struct nvme_remap_dev, sysdata);
}


/******** PCI configuration space **********/

/*
 * Helper macros for tweaking returned contents of PCI configuration space.
 *
 * value contains len bytes of data read from reg.
 * If fixup_reg is included in that range, fix up the contents of that
 * register to fixed_value.
 */
#define NR_FIX8(fixup_reg, fixed_value) do { \
		if (reg <= fixup_reg && fixup_reg < reg + len) \
			((u8 *) value)[fixup_reg - reg] = (u8) (fixed_value); \
	} while (0)

#define NR_FIX16(fixup_reg, fixed_value) do { \
		NR_FIX8(fixup_reg, fixed_value); \
		NR_FIX8(fixup_reg + 1, fixed_value >> 8); \
	} while (0)

#define NR_FIX24(fixup_reg, fixed_value) do { \
		NR_FIX8(fixup_reg, fixed_value); \
		NR_FIX8(fixup_reg + 1, fixed_value >> 8); \
		NR_FIX8(fixup_reg + 2, fixed_value >> 16); \
	} while (0)

#define NR_FIX32(fixup_reg, fixed_value) do { \
		NR_FIX16(fixup_reg, (u16) fixed_value); \
		NR_FIX16(fixup_reg + 2, fixed_value >> 16); \
	} while (0)

/*
 * Read PCI config space of the slot 0 (AHCI) device.
 * We pass through the read request to the underlying device, but
 * tweak the results in some cases.
 */
static int nvme_remap_pci_read_slot0(struct pci_bus *bus, int reg,
				     int len, u32 *value)
{
	struct nvme_remap_dev *nrdev = nrdev_from_bus(bus);
	struct pci_bus *ahci_dev_bus = nrdev->dev->bus;
	int ret;

	ret = ahci_dev_bus->ops->read(ahci_dev_bus, nrdev->dev->devfn,
				      reg, len, value);
	if (ret)
		return ret;

	/*
	 * Adjust the device class, to prevent this driver from attempting to
	 * additionally probe the device we're simulating here.
	 */
	NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_SATA_AHCI);

	/*
	 * Unset interrupt pin, otherwise ACPI tries to find routing
	 * info for our virtual IRQ, fails, and complains.
	 */
	NR_FIX8(PCI_INTERRUPT_PIN, 0);

	/*
	 * Truncate the AHCI BAR to not include the region that covers the
	 * hidden devices. This will cause the ahci driver to successfully
	 * probe th new device (instead of handing it over to this driver).
	 */
	if (nrdev->bar_sizing) {
		NR_FIX32(PCI_BASE_ADDRESS_5, ~(SZ_16K - 1));
		nrdev->bar_sizing = false;
	}

	return PCIBIOS_SUCCESSFUL;
}

/*
 * Read PCI config space of a remapped device.
 * Since the original PCI config space is inaccessible, we provide a minimal,
 * fake config space instead.
 */
static int nvme_remap_pci_read_remapped(struct pci_bus *bus, unsigned int port,
					int reg, int len, u32 *value)
{
	struct nvme_remap_dev *nrdev = nrdev_from_bus(bus);
	struct resource *remapped_mem;

	if (port > nrdev->num_remapped_devices)
		return PCIBIOS_DEVICE_NOT_FOUND;

	*value = 0;
	remapped_mem = &nrdev->remapped_dev_mem[port - 1];

	/* Set a Vendor ID, otherwise Linux assumes no device is present */
	NR_FIX16(PCI_VENDOR_ID, PCI_VENDOR_ID_INTEL);

	/* Always appear on & bus mastering */
	NR_FIX16(PCI_COMMAND, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);

	/* Set class so that nvme driver probes us */
	NR_FIX24(PCI_CLASS_PROG, PCI_CLASS_STORAGE_EXPRESS);

	if (nrdev->bar_sizing) {
		NR_FIX32(PCI_BASE_ADDRESS_0,
			 ~(resource_size(remapped_mem) - 1));
		nrdev->bar_sizing = false;
	} else {
		resource_size_t mem_start = remapped_mem->start;

		mem_start |= PCI_BASE_ADDRESS_MEM_TYPE_64;
		NR_FIX32(PCI_BASE_ADDRESS_0, mem_start);
		mem_start >>= 32;
		NR_FIX32(PCI_BASE_ADDRESS_1, mem_start);
	}

	return PCIBIOS_SUCCESSFUL;
}

/* Read PCI configuration space. */
static int nvme_remap_pci_read(struct pci_bus *bus, unsigned int devfn,
			       int reg, int len, u32 *value)
{
	if (PCI_SLOT(devfn) == 0)
		return nvme_remap_pci_read_slot0(bus, reg, len, value);
	else
		return nvme_remap_pci_read_remapped(bus, PCI_SLOT(devfn),
						    reg, len, value);
}

/*
 * Write PCI config space of the slot 0 (AHCI) device.
 * Apart from the special case of BAR sizing, we disable all writes.
 * Otherwise, the ahci driver could make changes (e.g. unset PCI bus master)
 * that would affect the operation of the NVMe devices.
 */
static int nvme_remap_pci_write_slot0(struct pci_bus *bus, int reg,
				      int len, u32 value)
{
	struct nvme_remap_dev *nrdev = nrdev_from_bus(bus);
	struct pci_bus *ahci_dev_bus = nrdev->dev->bus;

	if (reg >= PCI_BASE_ADDRESS_0 && reg <= PCI_BASE_ADDRESS_5) {
		/*
		 * Writing all-ones to a BAR means that the size of the
		 * memory region is being checked. Flag this so that we can
		 * reply with an appropriate size on the next read.
		 */
		if (value == ~0)
			nrdev->bar_sizing = true;

		return ahci_dev_bus->ops->write(ahci_dev_bus,
						nrdev->dev->devfn,
						reg, len, value);
	}

	return PCIBIOS_SET_FAILED;
}

/*
 * Write PCI config space of a remapped device.
 * Since the original PCI config space is inaccessible, we reject all
 * writes, except for the special case of BAR probing.
 */
static int nvme_remap_pci_write_remapped(struct pci_bus *bus,
					 unsigned int port,
					 int reg, int len, u32 value)
{
	struct nvme_remap_dev *nrdev = nrdev_from_bus(bus);

	if (port > nrdev->num_remapped_devices)
		return PCIBIOS_DEVICE_NOT_FOUND;

	/*
	 * Writing all-ones to a BAR means that the size of the memory
	 * region is being checked. Flag this so that we can reply with
	 * an appropriate size on the next read.
	 */
	if (value == ~0 && reg >= PCI_BASE_ADDRESS_0
			&& reg <= PCI_BASE_ADDRESS_5) {
		nrdev->bar_sizing = true;
		return PCIBIOS_SUCCESSFUL;
	}

	return PCIBIOS_SET_FAILED;
}

/* Write PCI configuration space. */
static int nvme_remap_pci_write(struct pci_bus *bus, unsigned int devfn,
				int reg, int len, u32 value)
{
	if (PCI_SLOT(devfn) == 0)
		return nvme_remap_pci_write_slot0(bus, reg, len, value);
	else
		return nvme_remap_pci_write_remapped(bus, PCI_SLOT(devfn),
						     reg, len, value);
}

static struct pci_ops nvme_remap_pci_ops = {
	.read	= nvme_remap_pci_read,
	.write	= nvme_remap_pci_write,
};


/******** Initialization & exit **********/

/*
 * Find a PCI domain ID to use for our fake bus.
 * Start at 0x10000 to not clash with ACPI _SEG domains (16 bits).
 */
static int find_free_domain(void)
{
	int domain = 0xffff;
	struct pci_bus *bus = NULL;

	while ((bus = pci_find_next_bus(bus)) != NULL)
		domain = max_t(int, domain, pci_domain_nr(bus));

	return domain + 1;
}

static int find_remapped_devices(struct nvme_remap_dev *nrdev,
				 struct list_head *resources)
{
	void __iomem *mmio;
	int i, count = 0;
	u32 cap;

	mmio = pcim_iomap(nrdev->dev, AHCI_PCI_BAR_STANDARD,
			  pci_resource_len(nrdev->dev,
					   AHCI_PCI_BAR_STANDARD));
	if (!mmio)
		return -ENODEV;

	/* Check if this device might have remapped nvme devices. */
	if (pci_resource_len(nrdev->dev, AHCI_PCI_BAR_STANDARD) < SZ_512K ||
	    !(readl(mmio + AHCI_VSCAP) & 1))
		return -ENODEV;

	cap = readq(mmio + AHCI_REMAP_CAP);
	for (i = AHCI_MAX_REMAP-1; i >= 0; i--) {
		struct resource *remapped_mem;

		if ((cap & (1 << i)) == 0)
			continue;
		if (readl(mmio + ahci_remap_dcc(i))
				!= PCI_CLASS_STORAGE_EXPRESS)
			continue;

		/* We've found a remapped device */
		remapped_mem = &nrdev->remapped_dev_mem[count++];
		remapped_mem->start =
			pci_resource_start(nrdev->dev, AHCI_PCI_BAR_STANDARD)
			+ ahci_remap_base(i);
		remapped_mem->end = remapped_mem->start
			+ AHCI_REMAP_N_SIZE - 1;
		remapped_mem->flags = IORESOURCE_MEM | IORESOURCE_PCI_FIXED;
		pci_add_resource(resources, remapped_mem);
	}

	pcim_iounmap(nrdev->dev, mmio);

	if (count == 0)
		return -ENODEV;

	nrdev->num_remapped_devices = count;
	dev_info(&nrdev->dev->dev, "Found %d remapped NVMe devices\n",
		 nrdev->num_remapped_devices);
	return 0;
}

static void nvme_remap_remove_root_bus(void *data)
{
	struct pci_bus *bus = data;

	pci_stop_root_bus(bus);
	pci_remove_root_bus(bus);
}

static int nvme_remap_probe(struct pci_dev *dev,
			    const struct pci_device_id *id)
{
	struct nvme_remap_dev *nrdev;
	LIST_HEAD(resources);
	int i;
	int ret;
	struct pci_dev *child;

	nrdev = devm_kzalloc(&dev->dev, sizeof(*nrdev), GFP_KERNEL);
	nrdev->sysdata.domain = find_free_domain();
	nrdev->sysdata.nvme_remap_dev = dev;
	nrdev->dev = dev;
	pci_set_drvdata(dev, nrdev);

	ret = pcim_enable_device(dev);
	if (ret < 0)
		return ret;

	pci_set_master(dev);

	ret = find_remapped_devices(nrdev, &resources);
	if (ret)
		return ret;

	/* Add resources from the original AHCI device */
	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
		struct resource *res = &dev->resource[i];

		if (res->start) {
			struct resource *nr_res = &nrdev->ahci_resources[i];

			nr_res->start = res->start;
			nr_res->end = res->end;
			nr_res->flags = res->flags;
			pci_add_resource(&resources, nr_res);
		}
	}

	/* Create virtual interrupts */
	nrdev->irq_base = devm_irq_alloc_descs(&dev->dev, -1, 0,
					       nrdev->num_remapped_devices + 1,
					       0);
	if (nrdev->irq_base < 0)
		return nrdev->irq_base;

	/* Create and populate PCI bus */
	nrdev->bus = pci_create_root_bus(&dev->dev, 0, &nvme_remap_pci_ops,
					 &nrdev->sysdata, &resources);
	if (!nrdev->bus)
		return -ENODEV;

	if (devm_add_action_or_reset(&dev->dev, nvme_remap_remove_root_bus,
				     nrdev->bus))
		return -ENOMEM;

	/* We don't support sharing MSI interrupts between these devices */
	nrdev->bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;

	pci_scan_child_bus(nrdev->bus);

	list_for_each_entry(child, &nrdev->bus->devices, bus_list) {
		/*
		 * Prevent PCI core from trying to move memory BARs around.
		 * The hidden NVMe devices are at fixed locations.
		 */
		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
			struct resource *res = &child->resource[i];

			if (res->flags & IORESOURCE_MEM)
				res->flags |= IORESOURCE_PCI_FIXED;
		}

		/* Share the legacy IRQ between all devices */
		child->irq = dev->irq;
	}

	pci_assign_unassigned_bus_resources(nrdev->bus);
	pci_bus_add_devices(nrdev->bus);

	return 0;
}

static const struct pci_device_id nvme_remap_ids[] = {
	/*
	 * Match all Intel RAID controllers.
	 *
	 * There's overlap here with the set of devices detected by the ahci
	 * driver, but ahci will only successfully probe when there
	 * *aren't* any remapped NVMe devices, and this driver will only
	 * successfully probe when there *are* remapped NVMe devices that
	 * need handling.
	 */
	{
		PCI_VDEVICE(INTEL, PCI_ANY_ID),
		.class = PCI_CLASS_STORAGE_RAID << 8,
		.class_mask = 0xffffff00,
	},
	{0,}
};
MODULE_DEVICE_TABLE(pci, nvme_remap_ids);

static struct pci_driver nvme_remap_drv = {
	.name		= MODULE_NAME,
	.id_table	= nvme_remap_ids,
	.probe		= nvme_remap_probe,
};
module_pci_driver(nvme_remap_drv);

MODULE_AUTHOR("Daniel Drake <drake@endlessm.com>");
MODULE_LICENSE("GPL v2");