IB/hfi1: Clean up hfi1_user_exp_rcv_setup function
Clean up hfi1_user_exp_rcv_setup function by moving page pinning and unpinning related code to separate functions. In order to reduce the number of parameters passed between functions, a new data structure struct tid_user_buf is defined and used. Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Harish Chegondi <harish.chegondi@intel.com> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com> Signed-off-by: Doug Ledford <dledford@redhat.com>
This commit is contained in:
Harish Chegondi
Doug Ledford
parent
7956371ea4
commit
9dc1170955
|
|
@ -75,20 +75,21 @@ struct tid_pageset {
|
|||
static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
|
||||
struct exp_tid_set *set,
|
||||
struct hfi1_filedata *fd);
|
||||
static u32 find_phys_blocks(struct page **pages, unsigned npages,
|
||||
struct tid_pageset *list);
|
||||
static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
|
||||
static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages);
|
||||
static int set_rcvarray_entry(struct hfi1_filedata *fd,
|
||||
struct tid_user_buf *tbuf,
|
||||
u32 rcventry, struct tid_group *grp,
|
||||
struct page **pages, unsigned npages);
|
||||
u16 pageidx, unsigned int npages);
|
||||
static int tid_rb_insert(void *arg, struct mmu_rb_node *node);
|
||||
static void cacheless_tid_rb_remove(struct hfi1_filedata *fdata,
|
||||
struct tid_rb_node *tnode);
|
||||
static void tid_rb_remove(void *arg, struct mmu_rb_node *node);
|
||||
static int tid_rb_invalidate(void *arg, struct mmu_rb_node *mnode);
|
||||
static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
|
||||
struct tid_group *grp, struct tid_pageset *sets,
|
||||
unsigned start, u16 count, struct page **pages,
|
||||
u32 *tidlist, unsigned *tididx, unsigned *pmapped);
|
||||
static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
|
||||
struct tid_group *grp,
|
||||
unsigned int start, u16 count,
|
||||
u32 *tidlist, unsigned int *tididx,
|
||||
unsigned int *pmapped);
|
||||
static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
|
||||
struct tid_group **grp);
|
||||
static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);
|
||||
|
|
@ -198,6 +199,92 @@ void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)
|
|||
fd->entry_to_rb = NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release pinned receive buffer pages.
|
||||
*
|
||||
* @mapped - true if the pages have been DMA mapped. false otherwise.
|
||||
* @idx - Index of the first page to unpin.
|
||||
* @npages - No of pages to unpin.
|
||||
*
|
||||
* If the pages have been DMA mapped (indicated by mapped parameter), their
|
||||
* info will be passed via a struct tid_rb_node. If they haven't been mapped,
|
||||
* their info will be passed via a struct tid_user_buf.
|
||||
*/
|
||||
static void unpin_rcv_pages(struct hfi1_filedata *fd,
|
||||
struct tid_user_buf *tidbuf,
|
||||
struct tid_rb_node *node,
|
||||
unsigned int idx,
|
||||
unsigned int npages,
|
||||
bool mapped)
|
||||
{
|
||||
struct page **pages;
|
||||
struct hfi1_devdata *dd = fd->uctxt->dd;
|
||||
|
||||
if (mapped) {
|
||||
pci_unmap_single(dd->pcidev, node->dma_addr,
|
||||
node->mmu.len, PCI_DMA_FROMDEVICE);
|
||||
pages = &node->pages[idx];
|
||||
} else {
|
||||
pages = &tidbuf->pages[idx];
|
||||
}
|
||||
hfi1_release_user_pages(fd->mm, pages, npages, mapped);
|
||||
fd->tid_n_pinned -= npages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pin receive buffer pages.
|
||||
*/
|
||||
static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)
|
||||
{
|
||||
int pinned;
|
||||
unsigned int npages;
|
||||
unsigned long vaddr = tidbuf->vaddr;
|
||||
struct page **pages = NULL;
|
||||
struct hfi1_devdata *dd = fd->uctxt->dd;
|
||||
|
||||
/* Get the number of pages the user buffer spans */
|
||||
npages = num_user_pages(vaddr, tidbuf->length);
|
||||
if (!npages)
|
||||
return -EINVAL;
|
||||
|
||||
if (npages > fd->uctxt->expected_count) {
|
||||
dd_dev_err(dd, "Expected buffer too big\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Verify that access is OK for the user buffer */
|
||||
if (!access_ok(VERIFY_WRITE, (void __user *)vaddr,
|
||||
npages * PAGE_SIZE)) {
|
||||
dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n",
|
||||
(void *)vaddr, npages);
|
||||
return -EFAULT;
|
||||
}
|
||||
/* Allocate the array of struct page pointers needed for pinning */
|
||||
pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
|
||||
if (!pages)
|
||||
return -ENOMEM;
|
||||
|
||||
/*
|
||||
* Pin all the pages of the user buffer. If we can't pin all the
|
||||
* pages, accept the amount pinned so far and program only that.
|
||||
* User space knows how to deal with partially programmed buffers.
|
||||
*/
|
||||
if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
|
||||
kfree(pages);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
|
||||
if (pinned <= 0) {
|
||||
kfree(pages);
|
||||
return pinned;
|
||||
}
|
||||
tidbuf->pages = pages;
|
||||
tidbuf->npages = npages;
|
||||
fd->tid_n_pinned += pinned;
|
||||
return pinned;
|
||||
}
|
||||
|
||||
/*
|
||||
* RcvArray entry allocation for Expected Receives is done by the
|
||||
* following algorithm:
|
||||
|
|
@ -253,62 +340,33 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
|
|||
int ret = 0, need_group = 0, pinned;
|
||||
struct hfi1_ctxtdata *uctxt = fd->uctxt;
|
||||
struct hfi1_devdata *dd = uctxt->dd;
|
||||
unsigned npages, ngroups, pageidx = 0, pageset_count, npagesets,
|
||||
unsigned int ngroups, pageidx = 0, pageset_count,
|
||||
tididx = 0, mapped, mapped_pages = 0;
|
||||
unsigned long vaddr = tinfo->vaddr;
|
||||
struct page **pages = NULL;
|
||||
u32 *tidlist = NULL;
|
||||
struct tid_pageset *pagesets = NULL;
|
||||
struct tid_user_buf *tidbuf;
|
||||
|
||||
/* Get the number of pages the user buffer spans */
|
||||
npages = num_user_pages(vaddr, tinfo->length);
|
||||
if (!npages)
|
||||
return -EINVAL;
|
||||
|
||||
if (npages > uctxt->expected_count) {
|
||||
dd_dev_err(dd, "Expected buffer too big\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
/* Verify that access is OK for the user buffer */
|
||||
if (!access_ok(VERIFY_WRITE, (void __user *)vaddr,
|
||||
npages * PAGE_SIZE)) {
|
||||
dd_dev_err(dd, "Fail vaddr %p, %u pages, !access_ok\n",
|
||||
(void *)vaddr, npages);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
pagesets = kcalloc(uctxt->expected_count, sizeof(*pagesets),
|
||||
GFP_KERNEL);
|
||||
if (!pagesets)
|
||||
tidbuf = kzalloc(sizeof(*tidbuf), GFP_KERNEL);
|
||||
if (!tidbuf)
|
||||
return -ENOMEM;
|
||||
|
||||
/* Allocate the array of struct page pointers needed for pinning */
|
||||
pages = kcalloc(npages, sizeof(*pages), GFP_KERNEL);
|
||||
if (!pages) {
|
||||
ret = -ENOMEM;
|
||||
goto bail;
|
||||
tidbuf->vaddr = tinfo->vaddr;
|
||||
tidbuf->length = tinfo->length;
|
||||
tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),
|
||||
GFP_KERNEL);
|
||||
if (!tidbuf->psets) {
|
||||
kfree(tidbuf);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pin all the pages of the user buffer. If we can't pin all the
|
||||
* pages, accept the amount pinned so far and program only that.
|
||||
* User space knows how to deal with partially programmed buffers.
|
||||
*/
|
||||
if (!hfi1_can_pin_pages(dd, fd->mm, fd->tid_n_pinned, npages)) {
|
||||
ret = -ENOMEM;
|
||||
goto bail;
|
||||
}
|
||||
|
||||
pinned = hfi1_acquire_user_pages(fd->mm, vaddr, npages, true, pages);
|
||||
pinned = pin_rcv_pages(fd, tidbuf);
|
||||
if (pinned <= 0) {
|
||||
ret = pinned;
|
||||
goto bail;
|
||||
kfree(tidbuf->psets);
|
||||
kfree(tidbuf);
|
||||
return pinned;
|
||||
}
|
||||
fd->tid_n_pinned += npages;
|
||||
|
||||
/* Find sets of physically contiguous pages */
|
||||
npagesets = find_phys_blocks(pages, pinned, pagesets);
|
||||
tidbuf->n_psets = find_phys_blocks(tidbuf, pinned);
|
||||
|
||||
/*
|
||||
* We don't need to access this under a lock since tid_used is per
|
||||
|
|
@ -316,10 +374,10 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
|
|||
* and hfi1_user_exp_rcv_setup() at the same time.
|
||||
*/
|
||||
spin_lock(&fd->tid_lock);
|
||||
if (fd->tid_used + npagesets > fd->tid_limit)
|
||||
if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)
|
||||
pageset_count = fd->tid_limit - fd->tid_used;
|
||||
else
|
||||
pageset_count = npagesets;
|
||||
pageset_count = tidbuf->n_psets;
|
||||
spin_unlock(&fd->tid_lock);
|
||||
|
||||
if (!pageset_count)
|
||||
|
|
@ -347,9 +405,9 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
|
|||
struct tid_group *grp =
|
||||
tid_group_pop(&uctxt->tid_group_list);
|
||||
|
||||
ret = program_rcvarray(fd, vaddr, grp, pagesets,
|
||||
ret = program_rcvarray(fd, tidbuf, grp,
|
||||
pageidx, dd->rcv_entries.group_size,
|
||||
pages, tidlist, &tididx, &mapped);
|
||||
tidlist, &tididx, &mapped);
|
||||
/*
|
||||
* If there was a failure to program the RcvArray
|
||||
* entries for the entire group, reset the grp fields
|
||||
|
|
@ -393,8 +451,8 @@ int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,
|
|||
unsigned use = min_t(unsigned, pageset_count - pageidx,
|
||||
grp->size - grp->used);
|
||||
|
||||
ret = program_rcvarray(fd, vaddr, grp, pagesets,
|
||||
pageidx, use, pages, tidlist,
|
||||
ret = program_rcvarray(fd, tidbuf, grp,
|
||||
pageidx, use, tidlist,
|
||||
&tididx, &mapped);
|
||||
if (ret < 0) {
|
||||
hfi1_cdbg(TID,
|
||||
|
|
@ -454,16 +512,14 @@ nomem:
|
|||
* If not everything was mapped (due to insufficient RcvArray entries,
|
||||
* for example), unpin all unmapped pages so we can pin them nex time.
|
||||
*/
|
||||
if (mapped_pages != pinned) {
|
||||
hfi1_release_user_pages(fd->mm, &pages[mapped_pages],
|
||||
pinned - mapped_pages,
|
||||
false);
|
||||
fd->tid_n_pinned -= pinned - mapped_pages;
|
||||
}
|
||||
if (mapped_pages != pinned)
|
||||
unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages,
|
||||
(pinned - mapped_pages), false);
|
||||
bail:
|
||||
kfree(pagesets);
|
||||
kfree(pages);
|
||||
kfree(tidbuf->psets);
|
||||
kfree(tidlist);
|
||||
kfree(tidbuf->pages);
|
||||
kfree(tidbuf);
|
||||
return ret > 0 ? 0 : ret;
|
||||
}
|
||||
|
||||
|
|
@ -553,11 +609,12 @@ int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static u32 find_phys_blocks(struct page **pages, unsigned npages,
|
||||
struct tid_pageset *list)
|
||||
static u32 find_phys_blocks(struct tid_user_buf *tidbuf, unsigned int npages)
|
||||
{
|
||||
unsigned pagecount, pageidx, setcount = 0, i;
|
||||
unsigned long pfn, this_pfn;
|
||||
struct page **pages = tidbuf->pages;
|
||||
struct tid_pageset *list = tidbuf->psets;
|
||||
|
||||
if (!npages)
|
||||
return 0;
|
||||
|
|
@ -620,13 +677,13 @@ static u32 find_phys_blocks(struct page **pages, unsigned npages,
|
|||
/**
|
||||
* program_rcvarray() - program an RcvArray group with receive buffers
|
||||
* @fd: filedata pointer
|
||||
* @vaddr: starting user virtual address
|
||||
* @tbuf: pointer to struct tid_user_buf that has the user buffer starting
|
||||
* virtual address, buffer length, page pointers, pagesets (array of
|
||||
* struct tid_pageset holding information on physically contiguous
|
||||
* chunks from the user buffer), and other fields.
|
||||
* @grp: RcvArray group
|
||||
* @sets: array of struct tid_pageset holding information on physically
|
||||
* contiguous chunks from the user buffer
|
||||
* @start: starting index into sets array
|
||||
* @count: number of struct tid_pageset's to program
|
||||
* @pages: an array of struct page * for the user buffer
|
||||
* @tidlist: the array of u32 elements when the information about the
|
||||
* programmed RcvArray entries is to be encoded.
|
||||
* @tididx: starting offset into tidlist
|
||||
|
|
@ -644,11 +701,11 @@ static u32 find_phys_blocks(struct page **pages, unsigned npages,
|
|||
* -ENOMEM or -EFAULT on error from set_rcvarray_entry(), or
|
||||
* number of RcvArray entries programmed.
|
||||
*/
|
||||
static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
|
||||
static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,
|
||||
struct tid_group *grp,
|
||||
struct tid_pageset *sets,
|
||||
unsigned start, u16 count, struct page **pages,
|
||||
u32 *tidlist, unsigned *tididx, unsigned *pmapped)
|
||||
unsigned int start, u16 count,
|
||||
u32 *tidlist, unsigned int *tididx,
|
||||
unsigned int *pmapped)
|
||||
{
|
||||
struct hfi1_ctxtdata *uctxt = fd->uctxt;
|
||||
struct hfi1_devdata *dd = uctxt->dd;
|
||||
|
|
@ -687,11 +744,11 @@ static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
|
|||
}
|
||||
|
||||
rcventry = grp->base + useidx;
|
||||
npages = sets[setidx].count;
|
||||
pageidx = sets[setidx].idx;
|
||||
npages = tbuf->psets[setidx].count;
|
||||
pageidx = tbuf->psets[setidx].idx;
|
||||
|
||||
ret = set_rcvarray_entry(fd, vaddr + (pageidx * PAGE_SIZE),
|
||||
rcventry, grp, pages + pageidx,
|
||||
ret = set_rcvarray_entry(fd, tbuf,
|
||||
rcventry, grp, pageidx,
|
||||
npages);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
|
@ -712,15 +769,17 @@ static int program_rcvarray(struct hfi1_filedata *fd, unsigned long vaddr,
|
|||
return idx;
|
||||
}
|
||||
|
||||
static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
|
||||
static int set_rcvarray_entry(struct hfi1_filedata *fd,
|
||||
struct tid_user_buf *tbuf,
|
||||
u32 rcventry, struct tid_group *grp,
|
||||
struct page **pages, unsigned npages)
|
||||
u16 pageidx, unsigned int npages)
|
||||
{
|
||||
int ret;
|
||||
struct hfi1_ctxtdata *uctxt = fd->uctxt;
|
||||
struct tid_rb_node *node;
|
||||
struct hfi1_devdata *dd = uctxt->dd;
|
||||
dma_addr_t phys;
|
||||
struct page **pages = tbuf->pages + pageidx;
|
||||
|
||||
/*
|
||||
* Allocate the node first so we can handle a potential
|
||||
|
|
@ -741,7 +800,7 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd, unsigned long vaddr,
|
|||
return -EFAULT;
|
||||
}
|
||||
|
||||
node->mmu.addr = vaddr;
|
||||
node->mmu.addr = tbuf->vaddr + (pageidx * PAGE_SIZE);
|
||||
node->mmu.len = npages * PAGE_SIZE;
|
||||
node->phys = page_to_phys(pages[0]);
|
||||
node->npages = npages;
|
||||
|
|
@ -820,10 +879,7 @@ static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
|
|||
*/
|
||||
hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0);
|
||||
|
||||
pci_unmap_single(dd->pcidev, node->dma_addr, node->mmu.len,
|
||||
PCI_DMA_FROMDEVICE);
|
||||
hfi1_release_user_pages(fd->mm, node->pages, node->npages, true);
|
||||
fd->tid_n_pinned -= node->npages;
|
||||
unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);
|
||||
|
||||
node->grp->used--;
|
||||
node->grp->map &= ~(1 << (node->rcventry - node->grp->base));
|
||||
|
|
|
|||
|
|
@ -51,6 +51,15 @@
|
|||
|
||||
#include "exp_rcv.h"
|
||||
|
||||
struct tid_user_buf {
|
||||
unsigned long vaddr;
|
||||
unsigned long length;
|
||||
unsigned int npages;
|
||||
struct page **pages;
|
||||
struct tid_pageset *psets;
|
||||
unsigned int n_psets;
|
||||
};
|
||||
|
||||
int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,
|
||||
struct hfi1_ctxtdata *uctxt);
|
||||
void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd);
|
||||
|
|
|
|||
Loading…
Reference in New Issue