123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367 |
- // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
- /*
- * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
- * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
- * Copyright 2019 Marvell. All rights reserved.
- */
- #include <linux/xarray.h>
- #include "uverbs.h"
- #include "core_priv.h"
- /**
- * rdma_umap_priv_init() - Initialize the private data of a vma
- *
- * @priv: The already allocated private data
- * @vma: The vm area struct that needs private data
- * @entry: entry into the mmap_xa that needs to be linked with
- * this vma
- *
- * Each time we map IO memory into user space this keeps track of the
- * mapping. When the device is hot-unplugged we 'zap' the mmaps in user space
- * to point to the zero page and allow the hot unplug to proceed.
- *
- * This is necessary for cases like PCI physical hot unplug as the actual BAR
- * memory may vanish after this and access to it from userspace could MCE.
- *
- * RDMA drivers supporting disassociation must have their user space designed
- * to cope in some way with their IO pages going to the zero page.
- *
- */
- void rdma_umap_priv_init(struct rdma_umap_priv *priv,
- struct vm_area_struct *vma,
- struct rdma_user_mmap_entry *entry)
- {
- struct ib_uverbs_file *ufile = vma->vm_file->private_data;
- priv->vma = vma;
- if (entry) {
- kref_get(&entry->ref);
- priv->entry = entry;
- }
- vma->vm_private_data = priv;
- /* vm_ops is setup in ib_uverbs_mmap() to avoid module dependencies */
- mutex_lock(&ufile->umap_lock);
- list_add(&priv->list, &ufile->umaps);
- mutex_unlock(&ufile->umap_lock);
- }
- EXPORT_SYMBOL(rdma_umap_priv_init);
- /**
- * rdma_user_mmap_io() - Map IO memory into a process
- *
- * @ucontext: associated user context
- * @vma: the vma related to the current mmap call
- * @pfn: pfn to map
- * @size: size to map
- * @prot: pgprot to use in remap call
- * @entry: mmap_entry retrieved from rdma_user_mmap_entry_get(), or NULL
- * if mmap_entry is not used by the driver
- *
- * This is to be called by drivers as part of their mmap() functions if they
- * wish to send something like PCI-E BAR memory to userspace.
- *
- * Return -EINVAL on wrong flags or size, -EAGAIN on failure to map. 0 on
- * success.
- */
- int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
- unsigned long pfn, unsigned long size, pgprot_t prot,
- struct rdma_user_mmap_entry *entry)
- {
- struct ib_uverbs_file *ufile = ucontext->ufile;
- struct rdma_umap_priv *priv;
- if (!(vma->vm_flags & VM_SHARED))
- return -EINVAL;
- if (vma->vm_end - vma->vm_start != size)
- return -EINVAL;
- /* Driver is using this wrong, must be called by ib_uverbs_mmap */
- if (WARN_ON(!vma->vm_file ||
- vma->vm_file->private_data != ufile))
- return -EINVAL;
- lockdep_assert_held(&ufile->device->disassociate_srcu);
- priv = kzalloc(sizeof(*priv), GFP_KERNEL);
- if (!priv)
- return -ENOMEM;
- vma->vm_page_prot = prot;
- if (io_remap_pfn_range(vma, vma->vm_start, pfn, size, prot)) {
- kfree(priv);
- return -EAGAIN;
- }
- rdma_umap_priv_init(priv, vma, entry);
- return 0;
- }
- EXPORT_SYMBOL(rdma_user_mmap_io);
- /**
- * rdma_user_mmap_entry_get_pgoff() - Get an entry from the mmap_xa
- *
- * @ucontext: associated user context
- * @pgoff: The mmap offset >> PAGE_SHIFT
- *
- * This function is called when a user tries to mmap with an offset (returned
- * by rdma_user_mmap_get_offset()) it initially received from the driver. The
- * rdma_user_mmap_entry was created by the function
- * rdma_user_mmap_entry_insert(). This function increases the refcnt of the
- * entry so that it won't be deleted from the xarray in the meantime.
- *
- * Return an reference to an entry if exists or NULL if there is no
- * match. rdma_user_mmap_entry_put() must be called to put the reference.
- */
- struct rdma_user_mmap_entry *
- rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
- unsigned long pgoff)
- {
- struct rdma_user_mmap_entry *entry;
- if (pgoff > U32_MAX)
- return NULL;
- xa_lock(&ucontext->mmap_xa);
- entry = xa_load(&ucontext->mmap_xa, pgoff);
- /*
- * If refcount is zero, entry is already being deleted, driver_removed
- * indicates that the no further mmaps are possible and we waiting for
- * the active VMAs to be closed.
- */
- if (!entry || entry->start_pgoff != pgoff || entry->driver_removed ||
- !kref_get_unless_zero(&entry->ref))
- goto err;
- xa_unlock(&ucontext->mmap_xa);
- ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#zx] returned\n",
- pgoff, entry->npages);
- return entry;
- err:
- xa_unlock(&ucontext->mmap_xa);
- return NULL;
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_get_pgoff);
- /**
- * rdma_user_mmap_entry_get() - Get an entry from the mmap_xa
- *
- * @ucontext: associated user context
- * @vma: the vma being mmap'd into
- *
- * This function is like rdma_user_mmap_entry_get_pgoff() except that it also
- * checks that the VMA is correct.
- */
- struct rdma_user_mmap_entry *
- rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
- struct vm_area_struct *vma)
- {
- struct rdma_user_mmap_entry *entry;
- if (!(vma->vm_flags & VM_SHARED))
- return NULL;
- entry = rdma_user_mmap_entry_get_pgoff(ucontext, vma->vm_pgoff);
- if (!entry)
- return NULL;
- if (entry->npages * PAGE_SIZE != vma->vm_end - vma->vm_start) {
- rdma_user_mmap_entry_put(entry);
- return NULL;
- }
- return entry;
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_get);
- static void rdma_user_mmap_entry_free(struct kref *kref)
- {
- struct rdma_user_mmap_entry *entry =
- container_of(kref, struct rdma_user_mmap_entry, ref);
- struct ib_ucontext *ucontext = entry->ucontext;
- unsigned long i;
- /*
- * Erase all entries occupied by this single entry, this is deferred
- * until all VMA are closed so that the mmap offsets remain unique.
- */
- xa_lock(&ucontext->mmap_xa);
- for (i = 0; i < entry->npages; i++)
- __xa_erase(&ucontext->mmap_xa, entry->start_pgoff + i);
- xa_unlock(&ucontext->mmap_xa);
- ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#zx] removed\n",
- entry->start_pgoff, entry->npages);
- if (ucontext->device->ops.mmap_free)
- ucontext->device->ops.mmap_free(entry);
- }
- /**
- * rdma_user_mmap_entry_put() - Drop reference to the mmap entry
- *
- * @entry: an entry in the mmap_xa
- *
- * This function is called when the mapping is closed if it was
- * an io mapping or when the driver is done with the entry for
- * some other reason.
- * Should be called after rdma_user_mmap_entry_get was called
- * and entry is no longer needed. This function will erase the
- * entry and free it if its refcnt reaches zero.
- */
- void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry)
- {
- kref_put(&entry->ref, rdma_user_mmap_entry_free);
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_put);
- /**
- * rdma_user_mmap_entry_remove() - Drop reference to entry and
- * mark it as unmmapable
- *
- * @entry: the entry to insert into the mmap_xa
- *
- * Drivers can call this to prevent userspace from creating more mappings for
- * entry, however existing mmaps continue to exist and ops->mmap_free() will
- * not be called until all user mmaps are destroyed.
- */
- void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry)
- {
- if (!entry)
- return;
- xa_lock(&entry->ucontext->mmap_xa);
- entry->driver_removed = true;
- xa_unlock(&entry->ucontext->mmap_xa);
- kref_put(&entry->ref, rdma_user_mmap_entry_free);
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_remove);
- /**
- * rdma_user_mmap_entry_insert_range() - Insert an entry to the mmap_xa
- * in a given range.
- *
- * @ucontext: associated user context.
- * @entry: the entry to insert into the mmap_xa
- * @length: length of the address that will be mmapped
- * @min_pgoff: minimum pgoff to be returned
- * @max_pgoff: maximum pgoff to be returned
- *
- * This function should be called by drivers that use the rdma_user_mmap
- * interface for implementing their mmap syscall A database of mmap offsets is
- * handled in the core and helper functions are provided to insert entries
- * into the database and extract entries when the user calls mmap with the
- * given offset. The function allocates a unique page offset in a given range
- * that should be provided to user, the user will use the offset to retrieve
- * information such as address to be mapped and how.
- *
- * Return: 0 on success and -ENOMEM on failure
- */
- int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
- struct rdma_user_mmap_entry *entry,
- size_t length, u32 min_pgoff,
- u32 max_pgoff)
- {
- struct ib_uverbs_file *ufile = ucontext->ufile;
- XA_STATE(xas, &ucontext->mmap_xa, min_pgoff);
- u32 xa_first, xa_last, npages;
- int err;
- u32 i;
- if (!entry)
- return -EINVAL;
- kref_init(&entry->ref);
- entry->ucontext = ucontext;
- /*
- * We want the whole allocation to be done without interruption from a
- * different thread. The allocation requires finding a free range and
- * storing. During the xa_insert the lock could be released, possibly
- * allowing another thread to choose the same range.
- */
- mutex_lock(&ufile->umap_lock);
- xa_lock(&ucontext->mmap_xa);
- /* We want to find an empty range */
- npages = (u32)DIV_ROUND_UP(length, PAGE_SIZE);
- entry->npages = npages;
- while (true) {
- /* First find an empty index */
- xas_find_marked(&xas, max_pgoff, XA_FREE_MARK);
- if (xas.xa_node == XAS_RESTART)
- goto err_unlock;
- xa_first = xas.xa_index;
- /* Is there enough room to have the range? */
- if (check_add_overflow(xa_first, npages, &xa_last))
- goto err_unlock;
- /*
- * Now look for the next present entry. If an entry doesn't
- * exist, we found an empty range and can proceed.
- */
- xas_next_entry(&xas, xa_last - 1);
- if (xas.xa_node == XAS_BOUNDS || xas.xa_index >= xa_last)
- break;
- }
- for (i = xa_first; i < xa_last; i++) {
- err = __xa_insert(&ucontext->mmap_xa, i, entry, GFP_KERNEL);
- if (err)
- goto err_undo;
- }
- /*
- * Internally the kernel uses a page offset, in libc this is a byte
- * offset. Drivers should not return pgoff to userspace.
- */
- entry->start_pgoff = xa_first;
- xa_unlock(&ucontext->mmap_xa);
- mutex_unlock(&ufile->umap_lock);
- ibdev_dbg(ucontext->device, "mmap: pgoff[%#lx] npages[%#x] inserted\n",
- entry->start_pgoff, npages);
- return 0;
- err_undo:
- for (; i > xa_first; i--)
- __xa_erase(&ucontext->mmap_xa, i - 1);
- err_unlock:
- xa_unlock(&ucontext->mmap_xa);
- mutex_unlock(&ufile->umap_lock);
- return -ENOMEM;
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_insert_range);
- /**
- * rdma_user_mmap_entry_insert() - Insert an entry to the mmap_xa.
- *
- * @ucontext: associated user context.
- * @entry: the entry to insert into the mmap_xa
- * @length: length of the address that will be mmapped
- *
- * This function should be called by drivers that use the rdma_user_mmap
- * interface for handling user mmapped addresses. The database is handled in
- * the core and helper functions are provided to insert entries into the
- * database and extract entries when the user calls mmap with the given offset.
- * The function allocates a unique page offset that should be provided to user,
- * the user will use the offset to retrieve information such as address to
- * be mapped and how.
- *
- * Return: 0 on success and -ENOMEM on failure
- */
- int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
- struct rdma_user_mmap_entry *entry,
- size_t length)
- {
- return rdma_user_mmap_entry_insert_range(ucontext, entry, length, 0,
- U32_MAX);
- }
- EXPORT_SYMBOL(rdma_user_mmap_entry_insert);
|