From: christophe lombard <clombard@linux.vnet.ibm.com>
To: linuxppc-dev@lists.ozlabs.org, fbarrat@linux.vnet.ibm.com,
ajd@linux.ibm.com, aik@ozlabs.ru, groug@kaod.org, clg@kaod.org
Subject: [PATCH 1/2] powerpc/powernv: Register IOMMU group for OpenCAPI devices
Date: Thu, 24 Oct 2019 15:28:04 +0200 [thread overview]
Message-ID: <20191024132805.30701-2-clombard@linux.vnet.ibm.com> (raw)
In-Reply-To: <20191024132805.30701-1-clombard@linux.vnet.ibm.com>
This patch adds group registration for the OpenCAPI devices.
An unique iommu group is register for multiple PE, ie for a set of
multiple devices sharing the same domain, same bus and same slot.
This groud registration will be used to assign an OpenCAPI device to a
guest to participate in VFIO, like vfio-pci.
The release_ownership hook is used to disable the Scheduled Process Area
and clean allocated data if it's not done previously when the ocxl driver
is unloaded.
To support multiple OpenCAPI devices on the same machine, iommu group
and platform data are declared in the npu_link which is common for each
devices sharing the same domain, same bus and same slot.
Signed-off-by: Christophe Lombard <clombard@linux.vnet.ibm.com>
---
arch/powerpc/platforms/powernv/ocxl.c | 164 +++++++++++++++++-----
arch/powerpc/platforms/powernv/pci-ioda.c | 19 ++-
arch/powerpc/platforms/powernv/pci.h | 13 ++
3 files changed, 156 insertions(+), 40 deletions(-)
diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c
index 12b146c2f855..67b2be965415 100644
--- a/arch/powerpc/platforms/powernv/ocxl.c
+++ b/arch/powerpc/platforms/powernv/ocxl.c
@@ -74,6 +74,8 @@ struct npu_link {
u16 fn_desired_actags[8];
struct actag_range fn_actags[8];
bool assignment_done;
+ struct iommu_group *group;
+ struct platform_data data;
};
static struct list_head links_list = LIST_HEAD_INIT(links_list);
static DEFINE_MUTEX(links_list_lock);
@@ -603,54 +605,56 @@ int pnv_ocxl_platform_setup(struct pci_dev *dev, int PE_mask,
{
struct pci_controller *hose = pci_bus_to_host(dev->bus);
struct pnv_phb *phb = hose->private_data;
- struct platform_data *data;
+ struct npu_link *link = NULL;
int xsl_irq;
u32 bdfn;
- int rc;
-
- data = kzalloc(sizeof(*data), GFP_KERNEL);
- if (!data)
- return -ENOMEM;
+ int rc = 0;
- rc = alloc_spa(dev, data);
- if (rc) {
- kfree(data);
- return rc;
+ mutex_lock(&links_list_lock);
+ link = find_link(dev);
+ if (!link) {
+ dev_err(&dev->dev, "Failed to setup platform\n");
+ mutex_unlock(&links_list_lock);
+ return -ENODEV;
}
+ rc = alloc_spa(dev, &link->data);
+ if (rc)
+ goto unlock;
+
rc = get_xsl_irq(dev, &xsl_irq);
if (rc) {
- free_spa(data);
- kfree(data);
- return rc;
+ free_spa(&link->data);
+ goto unlock;
}
- rc = map_xsl_regs(dev, &data->dsisr, &data->dar, &data->tfc,
- &data->pe_handle);
+ rc = map_xsl_regs(dev, &link->data.dsisr, &link->data.dar,
+ &link->data.tfc, &link->data.pe_handle);
if (rc) {
- free_spa(data);
- kfree(data);
- return rc;
+ free_spa(&link->data);
+ goto unlock;
}
bdfn = (dev->bus->number << 8) | dev->devfn;
rc = opal_npu_spa_setup(phb->opal_id, bdfn,
- virt_to_phys(data->spa->spa_mem),
+ virt_to_phys(link->data.spa->spa_mem),
PE_mask);
if (rc) {
dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc);
- unmap_xsl_regs(data->dsisr, data->dar, data->tfc,
- data->pe_handle);
- free_spa(data);
- kfree(data);
- return rc;
+ unmap_xsl_regs(link->data.dsisr, link->data.dar,
+ link->data.tfc, link->data.pe_handle);
+ free_spa(&link->data);
+ goto unlock;
}
- data->phb_opal_id = phb->opal_id;
- data->bdfn = bdfn;
- *platform_data = (void *) data;
+ link->data.phb_opal_id = phb->opal_id;
+ link->data.bdfn = bdfn;
*hwirq = xsl_irq;
- return 0;
+ *platform_data = (void *)&link->data;
+
+unlock:
+ mutex_unlock(&links_list_lock);
+ return rc;
}
EXPORT_SYMBOL_GPL(pnv_ocxl_platform_setup);
@@ -682,11 +686,13 @@ void pnv_ocxl_platform_release(void *platform_data)
struct platform_data *data = (struct platform_data *)platform_data;
int rc;
- rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
- WARN_ON(rc);
- unmap_xsl_regs(data->dsisr, data->dar, data->tfc, data->pe_handle);
- free_spa(data);
- kfree(data);
+ if (data->spa) {
+ rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
+ WARN_ON(rc);
+ unmap_xsl_regs(data->dsisr, data->dar, data->tfc,
+ data->pe_handle);
+ free_spa(data);
+ }
}
EXPORT_SYMBOL_GPL(pnv_ocxl_platform_release);
@@ -837,3 +843,95 @@ int pnv_ocxl_remove_pe(void *platform_data, int pasid, u32 *pid,
return remove_pe_from_cache(data, *pe_handle);
}
EXPORT_SYMBOL_GPL(pnv_ocxl_remove_pe);
+
+static void take_ownership(struct iommu_table_group *table_group)
+{
+}
+
+static void release_ownership(struct iommu_table_group *table_group)
+{
+ struct pnv_ioda_pe *pe = container_of(table_group,
+ struct pnv_ioda_pe,
+ table_group);
+ struct npu_link *link = NULL;
+
+ mutex_lock(&links_list_lock);
+
+ link = find_link(pe->pdev);
+ if (!link)
+ return;
+
+ if (link->data.spa)
+ pnv_ocxl_platform_release(&link->data);
+
+ mutex_unlock(&links_list_lock);
+}
+
+static long set_window(struct iommu_table_group *table_group,
+ int num, struct iommu_table *tbl)
+{
+ return 0;
+}
+
+static long unset_window(struct iommu_table_group *table_group,
+ int num)
+{
+ return 0;
+}
+
+static long create_table(struct iommu_table_group *table_group,
+ int num, __u32 page_shift, __u64 window_size,
+ __u32 levels, struct iommu_table **ptbl)
+{
+ return 0;
+}
+
+static struct iommu_table_group_ops pnv_ocxl_ops = {
+ .take_ownership = take_ownership,
+ .release_ownership = release_ownership,
+ .set_window = set_window,
+ .unset_window = unset_window,
+ .create_table = create_table,
+};
+
+static void group_release(void *iommu_data)
+{
+ struct iommu_table_group *table_group = iommu_data;
+
+ table_group->group = NULL;
+}
+
+struct iommu_table_group *pnv_ocxl_setup_table_group(struct pnv_ioda_pe *pe)
+{
+ struct iommu_table_group *table_group;
+ struct npu_link *link = NULL;
+ struct pci_controller *hose;
+
+ mutex_lock(&links_list_lock);
+
+ /* The functions of a device all share the same link and by
+ * default the same table group
+ */
+ link = find_link(pe->pdev);
+ if (!link)
+ return NULL;
+
+ hose = pe->phb->hose;
+ table_group = &pe->table_group;
+ table_group->ops = &pnv_ocxl_ops;
+ if (link->group) {
+ table_group->group = link->group;
+ iommu_group_set_iommudata(link->group, table_group,
+ group_release);
+ } else {
+ if (!table_group->group) {
+ iommu_register_group(table_group,
+ hose->global_number,
+ pe->pe_number);
+ link->group = table_group->group;
+ }
+ }
+
+ mutex_unlock(&links_list_lock);
+ return table_group;
+}
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index d8080558d020..3f98b05e2d55 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -2629,22 +2629,27 @@ static void pnv_pci_ioda_setup_iommu_api(void)
list_for_each_entry(hose, &hose_list, list_node) {
phb = hose->private_data;
- if (phb->type == PNV_PHB_NPU_NVLINK ||
- phb->type == PNV_PHB_NPU_OCAPI)
+ if (phb->type == PNV_PHB_NPU_NVLINK)
continue;
list_for_each_entry(pe, &phb->ioda.pe_list, list) {
struct iommu_table_group *table_group;
- table_group = pnv_try_setup_npu_table_group(pe);
- if (!table_group) {
- if (!pnv_pci_ioda_pe_dma_weight(pe))
+ if (phb->type == PNV_PHB_NPU_OCAPI) {
+ table_group = pnv_ocxl_setup_table_group(pe);
+ if (!table_group)
continue;
+ } else {
+ table_group = pnv_try_setup_npu_table_group(pe);
+ if (!table_group) {
+ if (!pnv_pci_ioda_pe_dma_weight(pe))
+ continue;
- table_group = &pe->table_group;
- iommu_register_group(&pe->table_group,
+ table_group = &pe->table_group;
+ iommu_register_group(&pe->table_group,
pe->phb->hose->global_number,
pe->pe_number);
+ }
}
pnv_ioda_setup_bus_iommu_group(pe, table_group,
pe->pbus);
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 469c24463247..df4b7583efea 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -218,6 +218,19 @@ extern struct iommu_table_group *pnv_try_setup_npu_table_group(
extern struct iommu_table_group *pnv_npu_compound_attach(
struct pnv_ioda_pe *pe);
+/* OpenCAPI functions */
+#if IS_ENABLED(CONFIG_OCXL_BASE)
+extern struct iommu_table_group *pnv_ocxl_setup_table_group(
+ struct pnv_ioda_pe *pe);
+#else
+static inline struct iommu_table_group *pnv_ocxl_setup_table_group(
+ struct pnv_ioda_pe *pe)
+{
+ return NULL;
+}
+#endif /* CONFIG_OCXL_BASE */
+
+
/* pci-ioda-tce.c */
#define POWERNV_IOMMU_DEFAULT_LEVELS 1
#define POWERNV_IOMMU_MAX_LEVELS 5
--
2.21.0
next prev parent reply other threads:[~2019-10-24 13:32 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-10-24 13:28 [PATCH 0/2] vfio pci: Add support for OpenCAPI devices christophe lombard
2019-10-24 13:28 ` christophe lombard [this message]
2019-10-24 13:28 ` [PATCH 2/2] vfio/pci: Introduce OpenCAPI devices support christophe lombard
2019-11-11 4:17 ` Alexey Kardashevskiy
2019-11-11 4:28 ` Andrew Donnellan
2019-11-12 2:15 ` Alexey Kardashevskiy
2019-10-24 14:32 ` [PATCH 0/2] vfio pci: Add support for OpenCAPI devices Greg Kurz
2019-11-11 3:55 ` Alexey Kardashevskiy
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191024132805.30701-2-clombard@linux.vnet.ibm.com \
--to=clombard@linux.vnet.ibm.com \
--cc=aik@ozlabs.ru \
--cc=ajd@linux.ibm.com \
--cc=clg@kaod.org \
--cc=fbarrat@linux.vnet.ibm.com \
--cc=groug@kaod.org \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.