mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-21 08:19:28 +00:00
![Christopher Yeoh](/assets/img/avatar_default.png)
The basic idea behind cross memory attach is to allow MPI programs doing intra-node communication to do a single copy of the message rather than a double copy of the message via shared memory. The following patch attempts to achieve this by allowing a destination process, given an address and size from a source process, to copy memory directly from the source process into its own address space via a system call. There is also a symmetrical ability to copy from the current process's address space into a destination process's address space. - Use of /proc/pid/mem has been considered, but there are issues with using it: - Does not allow for specifying iovecs for both src and dest, assuming preadv or pwritev was implemented either the area read from or written to would need to be contiguous. - Currently mem_read allows only processes who are currently ptrace'ing the target and are still able to ptrace the target to read from the target. This check could possibly be moved to the open call, but its not clear exactly what race this restriction is stopping (reason appears to have been lost) - Having to send the fd of /proc/self/mem via SCM_RIGHTS on unix domain socket is a bit ugly from a userspace point of view, especially when you may have hundreds if not (eventually) thousands of processes that all need to do this with each other - Doesn't allow for some future use of the interface we would like to consider adding in the future (see below) - Interestingly reading from /proc/pid/mem currently actually involves two copies! (But this could be fixed pretty easily) As mentioned previously use of vmsplice instead was considered, but has problems. Since you need the reader and writer working co-operatively if the pipe is not drained then you block. Which requires some wrapping to do non blocking on the send side or polling on the receive. In all to all communication it requires ordering otherwise you can deadlock. And in the example of many MPI tasks writing to one MPI task vmsplice serialises the copying. There are some cases of MPI collectives where even a single copy interface does not get us the performance gain we could. For example in an MPI_Reduce rather than copy the data from the source we would like to instead use it directly in a mathops (say the reduce is doing a sum) as this would save us doing a copy. We don't need to keep a copy of the data from the source. I haven't implemented this, but I think this interface could in the future do all this through the use of the flags - eg could specify the math operation and type and the kernel rather than just copying the data would apply the specified operation between the source and destination and store it in the destination. Although we don't have a "second user" of the interface (though I've had some nibbles from people who may be interested in using it for intra process messaging which is not MPI). This interface is something which hardware vendors are already doing for their custom drivers to implement fast local communication. And so in addition to this being useful for OpenMPI it would mean the driver maintainers don't have to fix things up when the mm changes. There was some discussion about how much faster a true zero copy would go. Here's a link back to the email with some testing I did on that: http://marc.info/?l=linux-mm&m=130105930902915&w=2 There is a basic man page for the proposed interface here: http://ozlabs.org/~cyeoh/cma/process_vm_readv.txt This has been implemented for x86 and powerpc, other architecture should mainly (I think) just need to add syscall numbers for the process_vm_readv and process_vm_writev. There are 32 bit compatibility versions for 64-bit kernels. For arch maintainers there are some simple tests to be able to quickly verify that the syscalls are working correctly here: http://ozlabs.org/~cyeoh/cma/cma-test-20110718.tgz Signed-off-by: Chris Yeoh <yeohc@au1.ibm.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Paul Mackerras <paulus@samba.org> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: David Howells <dhowells@redhat.com> Cc: James Morris <jmorris@namei.org> Cc: <linux-man@vger.kernel.org> Cc: <linux-arch@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1008 lines
22 KiB
C
1008 lines
22 KiB
C
/*
|
|
* linux/fs/read_write.c
|
|
*
|
|
* Copyright (C) 1991, 1992 Linus Torvalds
|
|
*/
|
|
|
|
#include <linux/slab.h>
|
|
#include <linux/stat.h>
|
|
#include <linux/fcntl.h>
|
|
#include <linux/file.h>
|
|
#include <linux/uio.h>
|
|
#include <linux/fsnotify.h>
|
|
#include <linux/security.h>
|
|
#include <linux/module.h>
|
|
#include <linux/syscalls.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/splice.h>
|
|
#include "read_write.h"
|
|
|
|
#include <asm/uaccess.h>
|
|
#include <asm/unistd.h>
|
|
|
|
const struct file_operations generic_ro_fops = {
|
|
.llseek = generic_file_llseek,
|
|
.read = do_sync_read,
|
|
.aio_read = generic_file_aio_read,
|
|
.mmap = generic_file_readonly_mmap,
|
|
.splice_read = generic_file_splice_read,
|
|
};
|
|
|
|
EXPORT_SYMBOL(generic_ro_fops);
|
|
|
|
static inline int unsigned_offsets(struct file *file)
|
|
{
|
|
return file->f_mode & FMODE_UNSIGNED_OFFSET;
|
|
}
|
|
|
|
static loff_t lseek_execute(struct file *file, struct inode *inode,
|
|
loff_t offset, loff_t maxsize)
|
|
{
|
|
if (offset < 0 && !unsigned_offsets(file))
|
|
return -EINVAL;
|
|
if (offset > maxsize)
|
|
return -EINVAL;
|
|
|
|
if (offset != file->f_pos) {
|
|
file->f_pos = offset;
|
|
file->f_version = 0;
|
|
}
|
|
return offset;
|
|
}
|
|
|
|
/**
|
|
* generic_file_llseek_size - generic llseek implementation for regular files
|
|
* @file: file structure to seek on
|
|
* @offset: file offset to seek to
|
|
* @origin: type of seek
|
|
* @size: max size of file system
|
|
*
|
|
* This is a variant of generic_file_llseek that allows passing in a custom
|
|
* file size.
|
|
*
|
|
* Synchronization:
|
|
* SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
|
|
* SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
|
|
* read/writes behave like SEEK_SET against seeks.
|
|
*/
|
|
loff_t
|
|
generic_file_llseek_size(struct file *file, loff_t offset, int origin,
|
|
loff_t maxsize)
|
|
{
|
|
struct inode *inode = file->f_mapping->host;
|
|
|
|
switch (origin) {
|
|
case SEEK_END:
|
|
offset += i_size_read(inode);
|
|
break;
|
|
case SEEK_CUR:
|
|
/*
|
|
* Here we special-case the lseek(fd, 0, SEEK_CUR)
|
|
* position-querying operation. Avoid rewriting the "same"
|
|
* f_pos value back to the file because a concurrent read(),
|
|
* write() or lseek() might have altered it
|
|
*/
|
|
if (offset == 0)
|
|
return file->f_pos;
|
|
/*
|
|
* f_lock protects against read/modify/write race with other
|
|
* SEEK_CURs. Note that parallel writes and reads behave
|
|
* like SEEK_SET.
|
|
*/
|
|
spin_lock(&file->f_lock);
|
|
offset = lseek_execute(file, inode, file->f_pos + offset,
|
|
maxsize);
|
|
spin_unlock(&file->f_lock);
|
|
return offset;
|
|
case SEEK_DATA:
|
|
/*
|
|
* In the generic case the entire file is data, so as long as
|
|
* offset isn't at the end of the file then the offset is data.
|
|
*/
|
|
if (offset >= i_size_read(inode))
|
|
return -ENXIO;
|
|
break;
|
|
case SEEK_HOLE:
|
|
/*
|
|
* There is a virtual hole at the end of the file, so as long as
|
|
* offset isn't i_size or larger, return i_size.
|
|
*/
|
|
if (offset >= i_size_read(inode))
|
|
return -ENXIO;
|
|
offset = i_size_read(inode);
|
|
break;
|
|
}
|
|
|
|
return lseek_execute(file, inode, offset, maxsize);
|
|
}
|
|
EXPORT_SYMBOL(generic_file_llseek_size);
|
|
|
|
/**
|
|
* generic_file_llseek - generic llseek implementation for regular files
|
|
* @file: file structure to seek on
|
|
* @offset: file offset to seek to
|
|
* @origin: type of seek
|
|
*
|
|
* This is a generic implemenation of ->llseek useable for all normal local
|
|
* filesystems. It just updates the file offset to the value specified by
|
|
* @offset and @origin under i_mutex.
|
|
*/
|
|
loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
|
|
{
|
|
struct inode *inode = file->f_mapping->host;
|
|
|
|
return generic_file_llseek_size(file, offset, origin,
|
|
inode->i_sb->s_maxbytes);
|
|
}
|
|
EXPORT_SYMBOL(generic_file_llseek);
|
|
|
|
/**
|
|
* noop_llseek - No Operation Performed llseek implementation
|
|
* @file: file structure to seek on
|
|
* @offset: file offset to seek to
|
|
* @origin: type of seek
|
|
*
|
|
* This is an implementation of ->llseek useable for the rare special case when
|
|
* userspace expects the seek to succeed but the (device) file is actually not
|
|
* able to perform the seek. In this case you use noop_llseek() instead of
|
|
* falling back to the default implementation of ->llseek.
|
|
*/
|
|
loff_t noop_llseek(struct file *file, loff_t offset, int origin)
|
|
{
|
|
return file->f_pos;
|
|
}
|
|
EXPORT_SYMBOL(noop_llseek);
|
|
|
|
loff_t no_llseek(struct file *file, loff_t offset, int origin)
|
|
{
|
|
return -ESPIPE;
|
|
}
|
|
EXPORT_SYMBOL(no_llseek);
|
|
|
|
loff_t default_llseek(struct file *file, loff_t offset, int origin)
|
|
{
|
|
struct inode *inode = file->f_path.dentry->d_inode;
|
|
loff_t retval;
|
|
|
|
mutex_lock(&inode->i_mutex);
|
|
switch (origin) {
|
|
case SEEK_END:
|
|
offset += i_size_read(inode);
|
|
break;
|
|
case SEEK_CUR:
|
|
if (offset == 0) {
|
|
retval = file->f_pos;
|
|
goto out;
|
|
}
|
|
offset += file->f_pos;
|
|
break;
|
|
case SEEK_DATA:
|
|
/*
|
|
* In the generic case the entire file is data, so as
|
|
* long as offset isn't at the end of the file then the
|
|
* offset is data.
|
|
*/
|
|
if (offset >= inode->i_size) {
|
|
retval = -ENXIO;
|
|
goto out;
|
|
}
|
|
break;
|
|
case SEEK_HOLE:
|
|
/*
|
|
* There is a virtual hole at the end of the file, so
|
|
* as long as offset isn't i_size or larger, return
|
|
* i_size.
|
|
*/
|
|
if (offset >= inode->i_size) {
|
|
retval = -ENXIO;
|
|
goto out;
|
|
}
|
|
offset = inode->i_size;
|
|
break;
|
|
}
|
|
retval = -EINVAL;
|
|
if (offset >= 0 || unsigned_offsets(file)) {
|
|
if (offset != file->f_pos) {
|
|
file->f_pos = offset;
|
|
file->f_version = 0;
|
|
}
|
|
retval = offset;
|
|
}
|
|
out:
|
|
mutex_unlock(&inode->i_mutex);
|
|
return retval;
|
|
}
|
|
EXPORT_SYMBOL(default_llseek);
|
|
|
|
loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
|
|
{
|
|
loff_t (*fn)(struct file *, loff_t, int);
|
|
|
|
fn = no_llseek;
|
|
if (file->f_mode & FMODE_LSEEK) {
|
|
if (file->f_op && file->f_op->llseek)
|
|
fn = file->f_op->llseek;
|
|
}
|
|
return fn(file, offset, origin);
|
|
}
|
|
EXPORT_SYMBOL(vfs_llseek);
|
|
|
|
SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, origin)
|
|
{
|
|
off_t retval;
|
|
struct file * file;
|
|
int fput_needed;
|
|
|
|
retval = -EBADF;
|
|
file = fget_light(fd, &fput_needed);
|
|
if (!file)
|
|
goto bad;
|
|
|
|
retval = -EINVAL;
|
|
if (origin <= SEEK_MAX) {
|
|
loff_t res = vfs_llseek(file, offset, origin);
|
|
retval = res;
|
|
if (res != (loff_t)retval)
|
|
retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
|
|
}
|
|
fput_light(file, fput_needed);
|
|
bad:
|
|
return retval;
|
|
}
|
|
|
|
#ifdef __ARCH_WANT_SYS_LLSEEK
|
|
SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
|
|
unsigned long, offset_low, loff_t __user *, result,
|
|
unsigned int, origin)
|
|
{
|
|
int retval;
|
|
struct file * file;
|
|
loff_t offset;
|
|
int fput_needed;
|
|
|
|
retval = -EBADF;
|
|
file = fget_light(fd, &fput_needed);
|
|
if (!file)
|
|
goto bad;
|
|
|
|
retval = -EINVAL;
|
|
if (origin > SEEK_MAX)
|
|
goto out_putf;
|
|
|
|
offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
|
|
origin);
|
|
|
|
retval = (int)offset;
|
|
if (offset >= 0) {
|
|
retval = -EFAULT;
|
|
if (!copy_to_user(result, &offset, sizeof(offset)))
|
|
retval = 0;
|
|
}
|
|
out_putf:
|
|
fput_light(file, fput_needed);
|
|
bad:
|
|
return retval;
|
|
}
|
|
#endif
|
|
|
|
|
|
/*
|
|
* rw_verify_area doesn't like huge counts. We limit
|
|
* them to something that fits in "int" so that others
|
|
* won't have to do range checks all the time.
|
|
*/
|
|
int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
|
|
{
|
|
struct inode *inode;
|
|
loff_t pos;
|
|
int retval = -EINVAL;
|
|
|
|
inode = file->f_path.dentry->d_inode;
|
|
if (unlikely((ssize_t) count < 0))
|
|
return retval;
|
|
pos = *ppos;
|
|
if (unlikely(pos < 0)) {
|
|
if (!unsigned_offsets(file))
|
|
return retval;
|
|
if (count >= -pos) /* both values are in 0..LLONG_MAX */
|
|
return -EOVERFLOW;
|
|
} else if (unlikely((loff_t) (pos + count) < 0)) {
|
|
if (!unsigned_offsets(file))
|
|
return retval;
|
|
}
|
|
|
|
if (unlikely(inode->i_flock && mandatory_lock(inode))) {
|
|
retval = locks_mandatory_area(
|
|
read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
|
|
inode, file, pos, count);
|
|
if (retval < 0)
|
|
return retval;
|
|
}
|
|
retval = security_file_permission(file,
|
|
read_write == READ ? MAY_READ : MAY_WRITE);
|
|
if (retval)
|
|
return retval;
|
|
return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
|
|
}
|
|
|
|
static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
|
|
{
|
|
set_current_state(TASK_UNINTERRUPTIBLE);
|
|
if (!kiocbIsKicked(iocb))
|
|
schedule();
|
|
else
|
|
kiocbClearKicked(iocb);
|
|
__set_current_state(TASK_RUNNING);
|
|
}
|
|
|
|
ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
|
|
{
|
|
struct iovec iov = { .iov_base = buf, .iov_len = len };
|
|
struct kiocb kiocb;
|
|
ssize_t ret;
|
|
|
|
init_sync_kiocb(&kiocb, filp);
|
|
kiocb.ki_pos = *ppos;
|
|
kiocb.ki_left = len;
|
|
kiocb.ki_nbytes = len;
|
|
|
|
for (;;) {
|
|
ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
|
|
if (ret != -EIOCBRETRY)
|
|
break;
|
|
wait_on_retry_sync_kiocb(&kiocb);
|
|
}
|
|
|
|
if (-EIOCBQUEUED == ret)
|
|
ret = wait_on_sync_kiocb(&kiocb);
|
|
*ppos = kiocb.ki_pos;
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(do_sync_read);
|
|
|
|
ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
|
|
{
|
|
ssize_t ret;
|
|
|
|
if (!(file->f_mode & FMODE_READ))
|
|
return -EBADF;
|
|
if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
|
|
return -EINVAL;
|
|
if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
|
|
return -EFAULT;
|
|
|
|
ret = rw_verify_area(READ, file, pos, count);
|
|
if (ret >= 0) {
|
|
count = ret;
|
|
if (file->f_op->read)
|
|
ret = file->f_op->read(file, buf, count, pos);
|
|
else
|
|
ret = do_sync_read(file, buf, count, pos);
|
|
if (ret > 0) {
|
|
fsnotify_access(file);
|
|
add_rchar(current, ret);
|
|
}
|
|
inc_syscr(current);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfs_read);
|
|
|
|
ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
|
|
{
|
|
struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
|
|
struct kiocb kiocb;
|
|
ssize_t ret;
|
|
|
|
init_sync_kiocb(&kiocb, filp);
|
|
kiocb.ki_pos = *ppos;
|
|
kiocb.ki_left = len;
|
|
kiocb.ki_nbytes = len;
|
|
|
|
for (;;) {
|
|
ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
|
|
if (ret != -EIOCBRETRY)
|
|
break;
|
|
wait_on_retry_sync_kiocb(&kiocb);
|
|
}
|
|
|
|
if (-EIOCBQUEUED == ret)
|
|
ret = wait_on_sync_kiocb(&kiocb);
|
|
*ppos = kiocb.ki_pos;
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(do_sync_write);
|
|
|
|
ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
|
|
{
|
|
ssize_t ret;
|
|
|
|
if (!(file->f_mode & FMODE_WRITE))
|
|
return -EBADF;
|
|
if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
|
|
return -EINVAL;
|
|
if (unlikely(!access_ok(VERIFY_READ, buf, count)))
|
|
return -EFAULT;
|
|
|
|
ret = rw_verify_area(WRITE, file, pos, count);
|
|
if (ret >= 0) {
|
|
count = ret;
|
|
if (file->f_op->write)
|
|
ret = file->f_op->write(file, buf, count, pos);
|
|
else
|
|
ret = do_sync_write(file, buf, count, pos);
|
|
if (ret > 0) {
|
|
fsnotify_modify(file);
|
|
add_wchar(current, ret);
|
|
}
|
|
inc_syscw(current);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfs_write);
|
|
|
|
static inline loff_t file_pos_read(struct file *file)
|
|
{
|
|
return file->f_pos;
|
|
}
|
|
|
|
static inline void file_pos_write(struct file *file, loff_t pos)
|
|
{
|
|
file->f_pos = pos;
|
|
}
|
|
|
|
SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
loff_t pos = file_pos_read(file);
|
|
ret = vfs_read(file, buf, count, &pos);
|
|
file_pos_write(file, pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
|
|
size_t, count)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
loff_t pos = file_pos_read(file);
|
|
ret = vfs_write(file, buf, count, &pos);
|
|
file_pos_write(file, pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
|
|
size_t count, loff_t pos)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
if (pos < 0)
|
|
return -EINVAL;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
ret = -ESPIPE;
|
|
if (file->f_mode & FMODE_PREAD)
|
|
ret = vfs_read(file, buf, count, &pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
|
|
asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
|
|
{
|
|
return SYSC_pread64((unsigned int) fd, (char __user *) buf,
|
|
(size_t) count, pos);
|
|
}
|
|
SYSCALL_ALIAS(sys_pread64, SyS_pread64);
|
|
#endif
|
|
|
|
SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
|
|
size_t count, loff_t pos)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
if (pos < 0)
|
|
return -EINVAL;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
ret = -ESPIPE;
|
|
if (file->f_mode & FMODE_PWRITE)
|
|
ret = vfs_write(file, buf, count, &pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
|
|
asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
|
|
{
|
|
return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
|
|
(size_t) count, pos);
|
|
}
|
|
SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
|
|
#endif
|
|
|
|
/*
|
|
* Reduce an iovec's length in-place. Return the resulting number of segments
|
|
*/
|
|
unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
|
|
{
|
|
unsigned long seg = 0;
|
|
size_t len = 0;
|
|
|
|
while (seg < nr_segs) {
|
|
seg++;
|
|
if (len + iov->iov_len >= to) {
|
|
iov->iov_len = to - len;
|
|
break;
|
|
}
|
|
len += iov->iov_len;
|
|
iov++;
|
|
}
|
|
return seg;
|
|
}
|
|
EXPORT_SYMBOL(iov_shorten);
|
|
|
|
ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
|
|
unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
|
|
{
|
|
struct kiocb kiocb;
|
|
ssize_t ret;
|
|
|
|
init_sync_kiocb(&kiocb, filp);
|
|
kiocb.ki_pos = *ppos;
|
|
kiocb.ki_left = len;
|
|
kiocb.ki_nbytes = len;
|
|
|
|
for (;;) {
|
|
ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
|
|
if (ret != -EIOCBRETRY)
|
|
break;
|
|
wait_on_retry_sync_kiocb(&kiocb);
|
|
}
|
|
|
|
if (ret == -EIOCBQUEUED)
|
|
ret = wait_on_sync_kiocb(&kiocb);
|
|
*ppos = kiocb.ki_pos;
|
|
return ret;
|
|
}
|
|
|
|
/* Do it by hand, with file-ops */
|
|
ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
|
|
unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
|
|
{
|
|
struct iovec *vector = iov;
|
|
ssize_t ret = 0;
|
|
|
|
while (nr_segs > 0) {
|
|
void __user *base;
|
|
size_t len;
|
|
ssize_t nr;
|
|
|
|
base = vector->iov_base;
|
|
len = vector->iov_len;
|
|
vector++;
|
|
nr_segs--;
|
|
|
|
nr = fn(filp, base, len, ppos);
|
|
|
|
if (nr < 0) {
|
|
if (!ret)
|
|
ret = nr;
|
|
break;
|
|
}
|
|
ret += nr;
|
|
if (nr != len)
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/* A write operation does a read from user space and vice versa */
|
|
#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
|
|
|
|
ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
|
|
unsigned long nr_segs, unsigned long fast_segs,
|
|
struct iovec *fast_pointer,
|
|
struct iovec **ret_pointer,
|
|
int check_access)
|
|
{
|
|
unsigned long seg;
|
|
ssize_t ret;
|
|
struct iovec *iov = fast_pointer;
|
|
|
|
/*
|
|
* SuS says "The readv() function *may* fail if the iovcnt argument
|
|
* was less than or equal to 0, or greater than {IOV_MAX}. Linux has
|
|
* traditionally returned zero for zero segments, so...
|
|
*/
|
|
if (nr_segs == 0) {
|
|
ret = 0;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* First get the "struct iovec" from user memory and
|
|
* verify all the pointers
|
|
*/
|
|
if (nr_segs > UIO_MAXIOV) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
if (nr_segs > fast_segs) {
|
|
iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
|
|
if (iov == NULL) {
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
}
|
|
if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
|
|
ret = -EFAULT;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* According to the Single Unix Specification we should return EINVAL
|
|
* if an element length is < 0 when cast to ssize_t or if the
|
|
* total length would overflow the ssize_t return value of the
|
|
* system call.
|
|
*
|
|
* Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
|
|
* overflow case.
|
|
*/
|
|
ret = 0;
|
|
for (seg = 0; seg < nr_segs; seg++) {
|
|
void __user *buf = iov[seg].iov_base;
|
|
ssize_t len = (ssize_t)iov[seg].iov_len;
|
|
|
|
/* see if we we're about to use an invalid len or if
|
|
* it's about to overflow ssize_t */
|
|
if (len < 0) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
if (check_access
|
|
&& unlikely(!access_ok(vrfy_dir(type), buf, len))) {
|
|
ret = -EFAULT;
|
|
goto out;
|
|
}
|
|
if (len > MAX_RW_COUNT - ret) {
|
|
len = MAX_RW_COUNT - ret;
|
|
iov[seg].iov_len = len;
|
|
}
|
|
ret += len;
|
|
}
|
|
out:
|
|
*ret_pointer = iov;
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t do_readv_writev(int type, struct file *file,
|
|
const struct iovec __user * uvector,
|
|
unsigned long nr_segs, loff_t *pos)
|
|
{
|
|
size_t tot_len;
|
|
struct iovec iovstack[UIO_FASTIOV];
|
|
struct iovec *iov = iovstack;
|
|
ssize_t ret;
|
|
io_fn_t fn;
|
|
iov_fn_t fnv;
|
|
|
|
if (!file->f_op) {
|
|
ret = -EINVAL;
|
|
goto out;
|
|
}
|
|
|
|
ret = rw_copy_check_uvector(type, uvector, nr_segs,
|
|
ARRAY_SIZE(iovstack), iovstack, &iov, 1);
|
|
if (ret <= 0)
|
|
goto out;
|
|
|
|
tot_len = ret;
|
|
ret = rw_verify_area(type, file, pos, tot_len);
|
|
if (ret < 0)
|
|
goto out;
|
|
|
|
fnv = NULL;
|
|
if (type == READ) {
|
|
fn = file->f_op->read;
|
|
fnv = file->f_op->aio_read;
|
|
} else {
|
|
fn = (io_fn_t)file->f_op->write;
|
|
fnv = file->f_op->aio_write;
|
|
}
|
|
|
|
if (fnv)
|
|
ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
|
|
pos, fnv);
|
|
else
|
|
ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
|
|
|
|
out:
|
|
if (iov != iovstack)
|
|
kfree(iov);
|
|
if ((ret + (type == READ)) > 0) {
|
|
if (type == READ)
|
|
fsnotify_access(file);
|
|
else
|
|
fsnotify_modify(file);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
|
|
unsigned long vlen, loff_t *pos)
|
|
{
|
|
if (!(file->f_mode & FMODE_READ))
|
|
return -EBADF;
|
|
if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
|
|
return -EINVAL;
|
|
|
|
return do_readv_writev(READ, file, vec, vlen, pos);
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfs_readv);
|
|
|
|
ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
|
|
unsigned long vlen, loff_t *pos)
|
|
{
|
|
if (!(file->f_mode & FMODE_WRITE))
|
|
return -EBADF;
|
|
if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
|
|
return -EINVAL;
|
|
|
|
return do_readv_writev(WRITE, file, vec, vlen, pos);
|
|
}
|
|
|
|
EXPORT_SYMBOL(vfs_writev);
|
|
|
|
SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
|
|
unsigned long, vlen)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
loff_t pos = file_pos_read(file);
|
|
ret = vfs_readv(file, vec, vlen, &pos);
|
|
file_pos_write(file, pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
if (ret > 0)
|
|
add_rchar(current, ret);
|
|
inc_syscr(current);
|
|
return ret;
|
|
}
|
|
|
|
SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
|
|
unsigned long, vlen)
|
|
{
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
loff_t pos = file_pos_read(file);
|
|
ret = vfs_writev(file, vec, vlen, &pos);
|
|
file_pos_write(file, pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
if (ret > 0)
|
|
add_wchar(current, ret);
|
|
inc_syscw(current);
|
|
return ret;
|
|
}
|
|
|
|
static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
|
|
{
|
|
#define HALF_LONG_BITS (BITS_PER_LONG / 2)
|
|
return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
|
|
}
|
|
|
|
SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
|
|
unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
|
|
{
|
|
loff_t pos = pos_from_hilo(pos_h, pos_l);
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
if (pos < 0)
|
|
return -EINVAL;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
ret = -ESPIPE;
|
|
if (file->f_mode & FMODE_PREAD)
|
|
ret = vfs_readv(file, vec, vlen, &pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
if (ret > 0)
|
|
add_rchar(current, ret);
|
|
inc_syscr(current);
|
|
return ret;
|
|
}
|
|
|
|
SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
|
|
unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
|
|
{
|
|
loff_t pos = pos_from_hilo(pos_h, pos_l);
|
|
struct file *file;
|
|
ssize_t ret = -EBADF;
|
|
int fput_needed;
|
|
|
|
if (pos < 0)
|
|
return -EINVAL;
|
|
|
|
file = fget_light(fd, &fput_needed);
|
|
if (file) {
|
|
ret = -ESPIPE;
|
|
if (file->f_mode & FMODE_PWRITE)
|
|
ret = vfs_writev(file, vec, vlen, &pos);
|
|
fput_light(file, fput_needed);
|
|
}
|
|
|
|
if (ret > 0)
|
|
add_wchar(current, ret);
|
|
inc_syscw(current);
|
|
return ret;
|
|
}
|
|
|
|
static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
|
|
size_t count, loff_t max)
|
|
{
|
|
struct file * in_file, * out_file;
|
|
struct inode * in_inode, * out_inode;
|
|
loff_t pos;
|
|
ssize_t retval;
|
|
int fput_needed_in, fput_needed_out, fl;
|
|
|
|
/*
|
|
* Get input file, and verify that it is ok..
|
|
*/
|
|
retval = -EBADF;
|
|
in_file = fget_light(in_fd, &fput_needed_in);
|
|
if (!in_file)
|
|
goto out;
|
|
if (!(in_file->f_mode & FMODE_READ))
|
|
goto fput_in;
|
|
retval = -ESPIPE;
|
|
if (!ppos)
|
|
ppos = &in_file->f_pos;
|
|
else
|
|
if (!(in_file->f_mode & FMODE_PREAD))
|
|
goto fput_in;
|
|
retval = rw_verify_area(READ, in_file, ppos, count);
|
|
if (retval < 0)
|
|
goto fput_in;
|
|
count = retval;
|
|
|
|
/*
|
|
* Get output file, and verify that it is ok..
|
|
*/
|
|
retval = -EBADF;
|
|
out_file = fget_light(out_fd, &fput_needed_out);
|
|
if (!out_file)
|
|
goto fput_in;
|
|
if (!(out_file->f_mode & FMODE_WRITE))
|
|
goto fput_out;
|
|
retval = -EINVAL;
|
|
in_inode = in_file->f_path.dentry->d_inode;
|
|
out_inode = out_file->f_path.dentry->d_inode;
|
|
retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
|
|
if (retval < 0)
|
|
goto fput_out;
|
|
count = retval;
|
|
|
|
if (!max)
|
|
max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
|
|
|
|
pos = *ppos;
|
|
if (unlikely(pos + count > max)) {
|
|
retval = -EOVERFLOW;
|
|
if (pos >= max)
|
|
goto fput_out;
|
|
count = max - pos;
|
|
}
|
|
|
|
fl = 0;
|
|
#if 0
|
|
/*
|
|
* We need to debate whether we can enable this or not. The
|
|
* man page documents EAGAIN return for the output at least,
|
|
* and the application is arguably buggy if it doesn't expect
|
|
* EAGAIN on a non-blocking file descriptor.
|
|
*/
|
|
if (in_file->f_flags & O_NONBLOCK)
|
|
fl = SPLICE_F_NONBLOCK;
|
|
#endif
|
|
retval = do_splice_direct(in_file, ppos, out_file, count, fl);
|
|
|
|
if (retval > 0) {
|
|
add_rchar(current, retval);
|
|
add_wchar(current, retval);
|
|
}
|
|
|
|
inc_syscr(current);
|
|
inc_syscw(current);
|
|
if (*ppos > max)
|
|
retval = -EOVERFLOW;
|
|
|
|
fput_out:
|
|
fput_light(out_file, fput_needed_out);
|
|
fput_in:
|
|
fput_light(in_file, fput_needed_in);
|
|
out:
|
|
return retval;
|
|
}
|
|
|
|
SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
|
|
{
|
|
loff_t pos;
|
|
off_t off;
|
|
ssize_t ret;
|
|
|
|
if (offset) {
|
|
if (unlikely(get_user(off, offset)))
|
|
return -EFAULT;
|
|
pos = off;
|
|
ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
|
|
if (unlikely(put_user(pos, offset)))
|
|
return -EFAULT;
|
|
return ret;
|
|
}
|
|
|
|
return do_sendfile(out_fd, in_fd, NULL, count, 0);
|
|
}
|
|
|
|
SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
|
|
{
|
|
loff_t pos;
|
|
ssize_t ret;
|
|
|
|
if (offset) {
|
|
if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
|
|
return -EFAULT;
|
|
ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
|
|
if (unlikely(put_user(pos, offset)))
|
|
return -EFAULT;
|
|
return ret;
|
|
}
|
|
|
|
return do_sendfile(out_fd, in_fd, NULL, count, 0);
|
|
}
|