123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862 |
- /*!
- *****************************************************************************
- * Copyright (c) Imagination Technologies Ltd.
- *
- * The contents of this file are subject to the MIT license as set out below.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * Alternatively, the contents of this file may be used under the terms of the
- * GNU General Public License Version 2 ("GPL")in which case the provisions of
- * GPL are applicable instead of those above.
- *
- * If you wish to allow use of your version of this file only under the terms
- * of GPL, and not to allow others to use your version of this file under the
- * terms of the MIT license, indicate your decision by deleting the provisions
- * above and replace them with the notice and other provisions required by GPL
- * as set out in the file called "GPLHEADER" included in this distribution. If
- * you do not delete the provisions above, a recipient may use your version of
- * this file under the terms of either the MIT license or GPL.
- *
- * This License is also included in this distribution in the file called
- * "MIT_COPYING".
- *
- *****************************************************************************/
- #include <linux/delay.h>
- #include <linux/interrupt.h>
- #include <linux/module.h>
- #include <linux/device.h>
- #include <linux/gfp.h>
- #include <linux/version.h>
- #if LINUX_VERSION_CODE < KERNEL_VERSION(5,10,0)
- #include <linux/dma-mapping.h>
- #else
- #include <linux/dma-map-ops.h>
- #endif
- #include <linux/pci.h>
- #include <linux/pm.h>
- #include <linux/pm_runtime.h>
- #include <linux/mod_devicetable.h>
- #include "uapi/version.h"
- #include "vha_common.h"
- #include "vha_plat.h"
- #define DEVICE_NAME "vha"
- #define IS_APOLLO_DEVICE(devid) ((devid) == PCI_APOLLO_DEVICE_ID)
- /*
- * from TCF Support FPGA.Technical Reference
- * Manual.1.0.92.Internal Atlas GEN.External.doc:
- */
- /* Altas - System control register bar */
- #define PCI_ATLAS_SYS_CTRL_REGS_BAR (0)
- /* Altas - System control register offset */
- #define PCI_ATLAS_SYS_CTRL_REGS_OFFSET (0x0000)
- /* Atlas - Offset of INTERRUPT_STATUS */
- /*#define PCI_ATLAS_INTERRUPT_STATUS (0x00E0)*/
- /* Atlas - Offset of INTERRUPT_ENABLE */
- /*#define PCI_ATLAS_INTERRUPT_ENABLE (0x00F0)*/
- /* Atlas - Offset of INTERRUPT_CLEAR */
- /*#define PCI_ATLAS_INTERRUPT_CLEAR (0x00F8)*/
- /* Atlas - Master interrupt enable */
- #define PCI_ATLAS_MASTER_ENABLE (1<<31)
- /* Atlas - Device interrupt */
- #define PCI_ATLAS_DEVICE_INT (1<<13)
- /* Atlas - SCB Logic soft reset */
- #define PCI_ATLAS_SCB_RESET (1<<4)
- /* Atlas - PDP2 soft reset */
- #define PCI_ATLAS_PDP2_RESET (1<<3)
- /* Atlas - PDP1 soft reset */
- #define PCI_ATLAS_PDP1_RESET (1<<2)
- /* Atlas - soft reset the DDR logic */
- #define PCI_ATLAS_DDR_RESET (1<<1)
- /* Atlas - soft reset the device under test */
- #define PCI_ATLAS_DUT_RESET (1<<0)
- #define PCI_ATLAS_RESET_REG_OFFSET (0x0080)
- #define PCI_ATLAS_RESET_BITS (PCI_ATLAS_DDR_RESET | PCI_ATLAS_DUT_RESET \
- | PCI_ATLAS_PDP1_RESET | PCI_ATLAS_PDP2_RESET | \
- PCI_ATLAS_SCB_RESET)
- /* Apollo - Offset of INTERRUPT_STATUS */
- #define PCI_APOLLO_INTERRUPT_STATUS (0x00C8)
- /* Apollo - Offset of INTERRUPT_ENABLE */
- #define PCI_APOLLO_INTERRUPT_ENABLE (0x00D8)
- /* Apollo - Offset of INTERRUPT_CLEAR */
- #define PCI_APOLLO_INTERRUPT_CLEAR (0x00E0)
- /* Apollo - DCM Logic soft reset */
- #define PCI_APOLLO_DCM_RESET (1<<10)
- #define PCI_APOLLO_RESET_BITS (PCI_ATLAS_RESET_BITS | PCI_APOLLO_DCM_RESET)
- #define PCI_ATLAS_TEST_CTRL (0xb0)
- #define PCI_APOLLO_TEST_CTRL (0x98)
- #define PCI_ATLAS_VENDOR_ID (0x1010)
- #define PCI_ATLAS_DEVICE_ID (0x1CF1)
- #define PCI_APOLLO_DEVICE_ID (0x1CF2)
- /* Number of core cycles used to measure the core clock frequency */
- #define FREQ_MEASURE_CYCLES 0x7fffff
- /*#define FPGA_IMAGE_REV_OFFSET (0x604)
- #define FPGA_IMAGE_REV_MASK (0xFFFF)*/
- /* Parameters applicable when using bus master mode */
- static unsigned long contig_phys_start;
- module_param(contig_phys_start, ulong, 0444);
- MODULE_PARM_DESC(contig_phys_start,
- "Physical address of start of contiguous region");
- static uint32_t contig_size;
- module_param(contig_size, uint, 0444);
- MODULE_PARM_DESC(contig_size,
- "Size of contiguous region: takes precedence over any PCI based memory");
- static uint32_t fpga_heap_type = IMG_MEM_HEAP_TYPE_UNIFIED;
- module_param(fpga_heap_type, uint, 0444);
- MODULE_PARM_DESC(fpga_heap_type, "Fpga primary heap type");
- static unsigned long pci_size;
- module_param(pci_size, ulong, 0444);
- MODULE_PARM_DESC(pci_size,
- "physical size in bytes, when 0 (default), use all memory in the PCI bar");
- static unsigned long pci_offset;
- module_param(pci_offset, ulong, 0444);
- MODULE_PARM_DESC(pci_offset, "offset from PCI bar start. (default: 0)");
- static bool mem_static_kptr = true;
- module_param(mem_static_kptr, bool, 0444);
- MODULE_PARM_DESC(mem_static_kptr,
- "Creates static kernel mapping for fpga memory");
- /*
- * Special handling (not implemented) is required for the VHA device
- * to be able to access both carvout buffers (internal memory) and
- * dmabuf buffers (system memory).The latter have to go through
- * the system bus to be accessed whereas the former do not.
- */
- static struct heap_config vha_plat_fpga_heap_configs[] = {
- /* Primary heap used for internal allocations */
- #ifdef FPGA_BUS_MASTERING
- {
- .type = -1, /* selected with fpga_heap_type */
- .options = {
- .unified.gfp_type = GFP_DMA32 | __GFP_ZERO,
- .unified.max_order = 4,
- },
- .to_dev_addr = NULL,
- .to_host_addr = NULL,
- },
- #elif CONFIG_GENERIC_ALLOCATOR
- {
- .type = IMG_MEM_HEAP_TYPE_CARVEOUT,
- /* .options.carveout to be filled at run time */
- /* .to_dev_addr to be filled at run time */
- /* .to_host_addr to be filled at run time */
- },
- #else
- #error Neither FPGA_BUS_MASTERING or CONFIG_GENERIC_ALLOCATOR was defined
- #endif
- /* Secondary heap used for importing an external memory */
- #ifdef FPGA_BUS_MASTERING
- {
- .type = IMG_MEM_HEAP_TYPE_ANONYMOUS,
- },
- #endif
- #if CONFIG_DMA_SHARED_BUFFER
- {
- .type = IMG_MEM_HEAP_TYPE_DMABUF,
- .to_dev_addr = NULL,
- #ifndef FPGA_BUS_MASTERING
- .options.dmabuf = {
- .use_sg_dma = true,
- },
- #endif
- },
- #endif
- };
- static const int vha_plat_fpga_heaps =
- sizeof(vha_plat_fpga_heap_configs)/sizeof(*vha_plat_fpga_heap_configs);
- static const struct pci_device_id pci_pci_ids[] = {
- { PCI_DEVICE(PCI_ATLAS_VENDOR_ID, PCI_ATLAS_DEVICE_ID), },
- { PCI_DEVICE(PCI_ATLAS_VENDOR_ID, PCI_APOLLO_DEVICE_ID), },
- { 0, }
- };
- MODULE_DEVICE_TABLE(pci, pci_pci_ids);
- struct imgpci_prvdata {
- int irq;
- struct {
- unsigned long addr;
- unsigned long size;
- void __iomem *km_addr;
- } memmap[3];
- struct pci_dev *pci_dev;
- };
- struct img_pci_driver {
- struct pci_dev *pci_dev;
- struct pci_driver pci_driver;
- };
- static int vha_plat_probe(struct pci_dev *pci_dev,
- const struct pci_device_id *id);
- static void vha_plat_remove(struct pci_dev *dev);
- static int vha_plat_suspend(struct device *dev);
- static int vha_plat_resume(struct device *dev);
- static int vha_plat_runtime_idle(struct device *dev);
- static int vha_plat_runtime_suspend(struct device *dev);
- static int vha_plat_runtime_resume(struct device *dev);
- static struct dev_pm_ops vha_pm_plat_ops = {
- #ifdef FPGA_BUS_MASTERING
- /* Runtime pm will not work with fpga internal memory
- * because pci bus driver suspend is also called,
- * which disables core/mem clocks */
- SET_RUNTIME_PM_OPS(vha_plat_runtime_suspend,
- vha_plat_runtime_resume, vha_plat_runtime_idle)
- #endif
- SET_SYSTEM_SLEEP_PM_OPS(vha_plat_suspend, vha_plat_resume)
- };
- static ssize_t info_show(struct device_driver *drv, char *buf)
- {
- return sprintf(buf, "VHA FPGA driver version : " VERSION_STRING "\n");
- }
- static DRIVER_ATTR_RO(info);
- static struct attribute *drv_attrs[] = {
- &driver_attr_info.attr,
- NULL
- };
- ATTRIBUTE_GROUPS(drv);
- static struct img_pci_driver vha_pci_drv = {
- .pci_driver = {
- .name = "vha_pci",
- .id_table = pci_pci_ids,
- .probe = vha_plat_probe,
- .remove = vha_plat_remove,
- .driver = {
- .pm = &vha_pm_plat_ops,
- .groups = drv_groups,
- }
- },
- };
- static ulong maxmapsizeMB = (sizeof(void *) == 4) ? 400 : 1024;
- static int interrupt_status_reg = -1;
- static int interrupt_clear_reg = -1;
- static int interrupt_enable_reg = -1;
- static int test_ctrl_reg = -1;
- static unsigned int fpga_readreg32(struct imgpci_prvdata *data,
- int bar, unsigned long offset
- )
- {
- void __iomem *reg =
- (void __iomem *)(data->memmap[bar].km_addr + offset);
- return ioread32(reg);
- }
- static void fpga_writereg32(struct imgpci_prvdata *data,
- int bar, unsigned long offset, int val)
- {
- void __iomem *reg =
- (void __iomem *)(data->memmap[bar].km_addr + offset);
- iowrite32(val, reg);
- }
- static void reset_fpga(struct pci_dev *dev,
- struct imgpci_prvdata *data, unsigned int mask)
- {
- uint32_t bits = 0;
- if (!dev)
- return;
- bits = PCI_APOLLO_RESET_BITS;
- dev_dbg(&dev->dev, "reset fpga!\n");
- bits &= mask;
- if (bits) {
- uint32_t val = fpga_readreg32(data, 0, PCI_ATLAS_RESET_REG_OFFSET);
- val &= ~bits;
- fpga_writereg32(data, 0, PCI_ATLAS_RESET_REG_OFFSET, val);
- udelay(100); /* arbitrary delays, just in case! */
- val |= bits;
- fpga_writereg32(data, 0, PCI_ATLAS_RESET_REG_OFFSET, val);
- /* If not only DUT is reset, add a delay */
- if (mask != PCI_ATLAS_DUT_RESET)
- msleep(100);
- else
- udelay(100); /* arbitrary delays, just in case! */
- }
- dev_dbg(&dev->dev, "reset fpga done!\n");
- }
- static void fpga_clear_irq(struct imgpci_prvdata *data, unsigned int intstatus)
- {
- unsigned int max_retries = 1000;
- while (fpga_readreg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_status_reg) && max_retries--)
- fpga_writereg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_clear_reg,
- (PCI_ATLAS_MASTER_ENABLE | intstatus));
- }
- static irqreturn_t pci_thread_irq(int irq, void *dev_id)
- {
- struct pci_dev *dev = (struct pci_dev *)dev_id;
- return vha_handle_thread_irq(&dev->dev);
- }
- static irqreturn_t pci_isrcb(int irq, void *dev_id)
- {
- unsigned int intstatus;
- struct pci_dev *dev = (struct pci_dev *)dev_id;
- struct imgpci_prvdata *data = vha_get_plat_data(&dev->dev);
- irqreturn_t ret = IRQ_NONE;
- if (data == NULL || dev_id == NULL) {
- /* spurious interrupt: not yet initialised. */
- goto exit;
- }
- intstatus = fpga_readreg32(data,
- PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_status_reg);
- if (intstatus) {
- ret = vha_handle_irq(&dev->dev);
- /*
- * We need to clear interrupts for the embedded device
- * via the fpga interrupt controller...
- */
- fpga_clear_irq(data, intstatus);
- } else {
- /* either a spurious interrupt, or, more likely
- * a shared interrupt line, which will be handled by another driver
- */
- goto exit;
- }
- exit:
- return ret;
- }
- /*
- * IO hooks : Address bus for hw registers is 32bit!
- */
- uint64_t vha_plat_read64(void *addr)
- {
- return (uint64_t)readl((const volatile void __iomem *)addr) |
- ((uint64_t)readl((const volatile void __iomem *)addr + 4) << 32);
- }
- void vha_plat_write64(void *addr, uint64_t val)
- {
- writel((uint32_t)(val & 0xffffffff), (volatile void __iomem *)addr);
- writel((uint32_t)(val >> 32), (volatile void __iomem *)addr + 4);
- }
- int vha_plat_deinit(void)
- {
- struct pci_dev *dev = vha_pci_drv.pci_dev;
- int ret;
- if (dev) {
- struct imgpci_prvdata *data = vha_get_plat_data(&dev->dev);
- if (data) {
- /* reset the hardware */
- reset_fpga(data->pci_dev, data, ~0);
- } else {
- dev_dbg(&dev->dev,
- "%s: prv data not found, HW reset omitted\n",
- __func__);
- }
- } else {
- pr_debug("%s: dev missing, HW reset omitted\n", __func__);
- }
- /* Unregister the driver from the OS */
- pci_unregister_driver(&(vha_pci_drv.pci_driver));
- ret = vha_deinit();
- if (ret)
- pr_err("VHA driver deinit failed\n");
- return ret;
- }
- #ifdef CONFIG_GENERIC_ALLOCATOR
- static phys_addr_t carveout_to_dev_addr(union heap_options *options,
- phys_addr_t addr)
- {
- phys_addr_t base = options->carveout.phys;
- size_t size = options->carveout.size;
- unsigned long offset = options->carveout.offs;
- if (addr - offset >= base && addr < base + size - offset)
- return addr - base;
- pr_err("%s: unexpected addr! base 0x%llx size %zu offs %zu addr 0x%llx\n",
- __func__, base, size, offset, addr);
- WARN_ON(1);
- return addr;
- }
- static phys_addr_t carveout_to_host_addr(union heap_options *options,
- phys_addr_t addr)
- {
- phys_addr_t base = options->carveout.phys;
- size_t size = options->carveout.size;
- unsigned long offset = options->carveout.offs;
- if (addr < size - offset)
- return base + addr;
- pr_err("%s: unexpected addr! base %llx size %zu offs %zu addr %#llx\n",
- __func__, base, size, offset, addr);
- WARN_ON(1);
- return addr;
- }
- static void *carveout_get_kptr(phys_addr_t addr,
- size_t size, enum img_mem_attr mattr)
- {
- /*
- * Device memory is I/O memory and as a rule, it cannot
- * be dereferenced safely without memory barriers, that
- * is why it is guarded by __iomem (return of ioremap)
- * and checked by sparse. It is accessed only through
- * ioread32(), iowrit32(), etc.
- *
- * In x86 this memory can be dereferenced and safely
- * accessed, i.e. a * __iomem pointer can be casted to
- * a regular void* * pointer. We cast this here
- * assuming FPGA is x86 and add __force to silence the
- * sparse warning
- *
- * Note: System memory carveout can be used with cached turned on.
- * */
- void *kptr = NULL;
- if (mattr & IMG_MEM_ATTR_UNCACHED)
- #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)
- kptr = (void * __force *)ioremap_nocache(addr, size);
- #else
- kptr = (void * __force *)ioremap(addr, size);
- #endif
- else if (mattr & IMG_MEM_ATTR_CACHED)
- kptr = (void * __force *)ioremap_cache(addr, size);
- else if (mattr & IMG_MEM_ATTR_WRITECOMBINE)
- kptr = (void * __force *)ioremap_wc(addr, size);
- return kptr;
- }
- static int carveout_put_kptr(void *addr)
- {
- iounmap((volatile void __iomem *)addr);
- return 0;
- }
- #endif
- static int vha_plat_probe(struct pci_dev *pci_dev,
- const struct pci_device_id *id)
- {
- int bar, ret = 0;
- struct imgpci_prvdata *data;
- size_t maxmapsize = maxmapsizeMB * 1024 * 1024;
- struct device *dev = &pci_dev->dev;
- int heap;
- dev_dbg(dev, "probing device, pci_dev\n");
- /* Enable the device */
- if (pci_enable_device(pci_dev))
- goto out_free;
- if (dev->dma_mask) {
- dev_info(dev, "%s dev->dma_mask : %#llx\n",
- __func__, *dev->dma_mask);
- } else {
- dev_info(dev, "%s mask unset, setting coherent\n", __func__);
- dev->dma_mask = &dev->coherent_dma_mask;
- }
- dev_info(dev, "%s dma_set_mask %#llx\n", __func__, dma_get_mask(dev));
- ret = dma_set_mask(dev, dma_get_mask(dev));
- if (ret) {
- dev_err(dev, "%s failed to set dma mask\n", __func__);
- goto out_disable;
- }
- /* Reserve PCI I/O and memory resources */
- if (pci_request_regions(pci_dev, "imgpci"))
- goto out_disable;
- /* Create a kernel space mapping for each of the bars */
- data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
- dev_dbg(dev, "allocated imgpci_prvdata @ %p\n", data);
- memset(data, 0, sizeof(*data));
- for (bar = 0; bar < 3; bar++) {
- data->memmap[bar].size = pci_resource_len(pci_dev, bar);
- data->memmap[bar].addr = pci_resource_start(pci_dev, bar);
- if (bar == 2) {
- if (pci_size)
- data->memmap[bar].size = pci_size;
- /* ioremap fpga memory only when static mode is used */
- if (!mem_static_kptr)
- continue;
- }
- if (data->memmap[bar].size > maxmapsize) {
- /*
- * We avoid mapping too big regions: we do not need
- * such a big amount of memory and some times we do
- * not have enough contiguous 'vmallocable' memory.
- */
- dev_warn(dev, "not mapping all mem for bar %u\n", bar);
- data->memmap[bar].size = maxmapsize;
- }
- data->memmap[bar].km_addr = devm_ioremap(dev,
- pci_resource_start(pci_dev, bar),
- data->memmap[bar].size);
- dev_dbg(dev, "[bar %u] addr: 0x%lx size: 0x%lx km: 0x%p\n",
- bar, data->memmap[bar].addr,
- data->memmap[bar].size,
- data->memmap[bar].km_addr);
- }
- /* Get the IRQ...*/
- data->irq = pci_dev->irq;
- data->pci_dev = pci_dev;
- vha_pci_drv.pci_dev = pci_dev;
- reset_fpga(pci_dev, data, ~0);
- interrupt_status_reg = PCI_ATLAS_SYS_CTRL_REGS_OFFSET +
- PCI_APOLLO_INTERRUPT_STATUS;
- interrupt_clear_reg = PCI_ATLAS_SYS_CTRL_REGS_OFFSET +
- PCI_APOLLO_INTERRUPT_CLEAR;
- interrupt_enable_reg = PCI_ATLAS_SYS_CTRL_REGS_OFFSET +
- PCI_APOLLO_INTERRUPT_ENABLE;
- test_ctrl_reg = PCI_ATLAS_SYS_CTRL_REGS_OFFSET +
- PCI_APOLLO_TEST_CTRL;
- /*
- * We need to enable interrupts for the embedded device
- * via the fpga interrupt controller...
- */
- {
- unsigned int ena;
- ena = fpga_readreg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_enable_reg);
- ena |= PCI_ATLAS_MASTER_ENABLE | PCI_ATLAS_DEVICE_INT;
- fpga_writereg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_enable_reg, ena);
- fpga_clear_irq(data, ena);
- }
- #ifdef FPGA_BUS_MASTERING
- dev_dbg(dev, "enabling FPGA bus mastering\n");
- fpga_writereg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR, test_ctrl_reg, 0x0);
- #else
- /* Route to internal RAM - this is reset value */
- dev_dbg(dev, "disabling FPGA bus mastering\n");
- fpga_writereg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR, test_ctrl_reg, 0x1);
- #endif
- /* patch heap config with PCI memory addresses */
- for (heap = 0; heap < vha_plat_fpga_heaps; heap++) {
- struct heap_config *cfg = &vha_plat_fpga_heap_configs[heap];
- #ifdef CONFIG_GENERIC_ALLOCATOR
- if (cfg->type == IMG_MEM_HEAP_TYPE_CARVEOUT) {
- if (contig_size && contig_phys_start) {
- /*
- * 2 types of carveout memory are supported:
- * memory carved out of the main DDR
- * memory region.
- * eg: linux boot option memmap=512M$0x5CAFFFFF
- * This is configured using module parameters:
- * contig_phys_start and size
- * DDR populated in the actual PCI card,
- * in BAR 4.
- * The module parameters take precedence
- * over PCI memory.
- */
- cfg->options.carveout.phys = contig_phys_start;
- cfg->options.carveout.size = contig_size;
- cfg->to_dev_addr = NULL;
- cfg->to_host_addr = NULL;
- dev_info(dev, "using %dMB CARVEOUT at x%lx\n",
- contig_size/1024/1024,
- contig_phys_start);
- } else {
- cfg->options.carveout.phys =
- data->memmap[2].addr;
- if (mem_static_kptr)
- cfg->options.carveout.kptr =
- data->memmap[2].km_addr;
- cfg->options.carveout.size =
- data->memmap[2].size;
- cfg->options.carveout.offs = pci_offset;
- cfg->to_dev_addr = carveout_to_dev_addr;
- cfg->to_host_addr = carveout_to_host_addr;
- dev_info(dev, "using %zuMB CARVEOUT from PCI at 0x%llx\n",
- cfg->options.carveout.size/1024/1024,
- cfg->options.carveout.phys);
- }
- /* IO memory access callbacks */
- if (!mem_static_kptr) {
- /* Dynamic kernel memory mapping */
- cfg->options.carveout.get_kptr = carveout_get_kptr;
- cfg->options.carveout.put_kptr = carveout_put_kptr;
- }
- break;
- }
- #endif
- if (cfg->type == IMG_MEM_HEAP_TYPE_COHERENT) {
- ret = dma_declare_coherent_memory(dev,
- contig_phys_start, contig_phys_start,
- contig_size
- #if LINUX_VERSION_CODE < KERNEL_VERSION(5,1,0)
- #if LINUX_VERSION_CODE < KERNEL_VERSION(4,14,0)
- , DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE
- #else
- , DMA_MEMORY_EXCLUSIVE
- #endif
- #endif
- );
- if (ret == 0) {
- dev_err(dev, "failed to initialize coherent memory!\n");
- /*
- * We will fallback to the default pool anyway
- * goto out_release;
- */
- }
- break;
- }
- }
- #ifdef FPGA_BUS_MASTERING
- /* Allow the core driver to control pm_runtime */
- pm_runtime_allow(dev);
- #endif
- ret = vha_add_dev(dev, vha_plat_fpga_heap_configs,
- vha_plat_fpga_heaps, data,
- data->memmap[1].km_addr, data->memmap[1].size);
- if (ret) {
- dev_err(dev, "failed to initialize driver core!\n");
- goto out_heap_deinit;
- }
- /*
- * Reset FPGA DUT only after disabling clocks in
- * vha_add_dev()-> get properties.
- * This workaround is required to ensure that
- * clocks (on daughter board) are enabled for test slave scripts to
- * read FPGA build version register.
- * NOTE: Asserting other bits like DDR reset bit cause problems
- * with bus mastering feature, thus results in memory failures.
- */
- reset_fpga(pci_dev, data, PCI_ATLAS_DUT_RESET);
- {
- /*uint32_t fpga_rev = fpga_readreg32(data, 1,
- FPGA_IMAGE_REV_OFFSET) & FPGA_IMAGE_REV_MASK;
- dev_dbg(dev, "fpga image revision: 0x%x\n", fpga_rev);
- if (!fpga_rev || fpga_rev == 0xdead1) {
- dev_err(dev, "fpga revision incorrect (0x%x)!\n",
- fpga_rev);
- goto out_rm_dev;
- }*/
- }
- /* Install the ISR callback...*/
- ret = devm_request_threaded_irq(dev, data->irq, &pci_isrcb,
- &pci_thread_irq, IRQF_SHARED, DEVICE_NAME,
- (void *)pci_dev);
- if (ret) {
- dev_err(dev, "failed to request irq!\n");
- goto out_rm_dev;
- }
- dev_dbg(dev, "registered irq %d\n", data->irq);
- /* Try to calibrate the core if needed */
- ret = vha_dev_calibrate(dev, FREQ_MEASURE_CYCLES);
- if (ret) {
- dev_err(dev, "%s: Failed to start clock calibration!\n", __func__);
- goto out_rm_dev;
- }
- return ret;
- out_rm_dev:
- vha_rm_dev(dev);
- out_heap_deinit:
- #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
- /* Release any declared mem regions */
- dma_release_declared_memory(dev);
- #endif
- /*out_release:*/
- pci_release_regions(pci_dev);
- out_disable:
- pci_disable_device(pci_dev);
- out_free:
- return ret;
- }
- static void vha_plat_remove(struct pci_dev *dev)
- {
- struct imgpci_prvdata *data = vha_get_plat_data(&dev->dev);
- dev_dbg(&dev->dev, "removing device\n");
- if (data == NULL) {
- dev_err(&dev->dev, "PCI priv data missing!\n");
- } else {
- /*
- * We need to disable interrupts for the
- * embedded device via the fpga interrupt controller...
- */
- fpga_writereg32(data, PCI_ATLAS_SYS_CTRL_REGS_BAR,
- interrupt_enable_reg, 0x00000000);
- #ifdef FPGA_BUS_MASTERING
- /* Route to internal RAM - this is reset value */
- dev_dbg(&dev->dev, "disabling FPGA bus mastering\n");
- fpga_writereg32(data,
- PCI_ATLAS_SYS_CTRL_REGS_BAR,
- test_ctrl_reg, 0x1);
- #endif
- }
- #if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
- /* Release any declared mem regions */
- dma_release_declared_memory(&dev->dev);
- #endif
- pci_release_regions(dev);
- pci_disable_device(dev);
- vha_rm_dev(&dev->dev);
- #ifdef FPGA_BUS_MASTERING
- pm_runtime_forbid(&dev->dev);
- #endif
- }
- #ifdef CONFIG_PM
- static int vha_plat_suspend(struct device *dev)
- {
- struct pci_dev *pci_dev = vha_pci_drv.pci_dev;
- struct imgpci_prvdata *data = vha_get_plat_data(dev);
- int ret;
- ret = vha_suspend_dev(dev);
- if (ret) {
- dev_dbg(dev, "suspend device\n");
- reset_fpga(pci_dev, data, PCI_ATLAS_DUT_RESET);
- } else
- dev_err(dev, "failed to suspend!\n");
- return ret;
- }
- static int vha_plat_resume(struct device *dev)
- {
- struct pci_dev *pci_dev = vha_pci_drv.pci_dev;
- struct imgpci_prvdata *data = vha_get_plat_data(dev);
- int ret;
- dev_dbg(dev, "resume device\n");
- reset_fpga(pci_dev, data, PCI_ATLAS_DUT_RESET);
- ret = vha_resume_dev(dev);
- if (ret)
- dev_err(dev, "failed to resume!\n");
- return ret;
- }
- static int __maybe_unused vha_plat_runtime_idle(struct device *dev)
- {
- dev_dbg(dev, "%s\n", __func__);
- return 0;
- }
- static int __maybe_unused vha_plat_runtime_suspend(struct device *dev)
- {
- dev_dbg(dev, "%s\n", __func__);
- return 0;
- }
- static int __maybe_unused vha_plat_runtime_resume(struct device *dev)
- {
- dev_dbg(dev, "%s\n", __func__);
- return 0;
- }
- #endif
- int vha_plat_init(void)
- {
- int ret;
- #ifdef FPGA_BUS_MASTERING
- vha_plat_fpga_heap_configs[0].type = fpga_heap_type;
- #endif
- ret = pci_register_driver(&vha_pci_drv.pci_driver);
- if (ret) {
- pr_err("failed to register PCI driver!\n");
- return ret;
- }
- /* pci_dev should be set in probe */
- if (!vha_pci_drv.pci_dev) {
- pr_err("failed to find VHA PCI dev!\n");
- pci_unregister_driver(&vha_pci_drv.pci_driver);
- return -ENODEV;
- }
- return 0;
- }
|