fuse: allow parallel direct writes past EOF

Extending FOPEN_PARALLEL_DIRECT_WRITES writes were forced onto the
exclusive inode lock, re-serializing the parallel phase. The exclusive
lock only bundled "write + advance i_size + undo-on-failure" into one
unit. But i_size is committed by fuse_write_update_attr() under
fi->lock, only on a successful growing write and independent of the
inode rwsem -- so shared-lock writers commit size correctly and have
nothing to undo. Drop the past-EOF exclusive triggers and gate the
whole-file fuse_do_truncate() rollback on holding the exclusive lock.

Lock mode is passed to __fuse_direct_IO(); i_size is committed at the
same point in every path, only the failure rollback differs:

  non-exclusive (relaxed, parallel):
    fuse_direct_write_iter
      fuse_dio_lock -> inode_lock_shared       (exclusive=false)
      __fuse_direct_IO(.., false)
        fuse_direct_io()          write to server
        fuse_write_update_attr()  commit i_size (on success)
        no rollback

  exclusive (append / caching / !parallel):
    fuse_direct_write_iter
      fuse_dio_lock -> inode_lock              (exclusive=true)
      __fuse_direct_IO(.., true)
        fuse_direct_io()          write to server
        fuse_write_update_attr()  commit i_size (on success)
        ret<0 & extend -> fuse_do_truncate()  rollback

  exclusive (caching-mode O_DIRECT):
    fuse_cache_write_iter -> inode_lock (exclusive)
      generic_file_direct_write -> fuse_direct_IO
        __fuse_direct_IO(.., true)
          fuse_direct_io()          write to server
          fuse_write_update_attr()  commit i_size (on success)
          ret<0 & extend -> fuse_do_truncate()  rollback

Signed-off-by: Bernd Schubert <bschubert@ddn.com>

Author: bsbernd

fs/fuse/file.c

@@ -1560,13 +1560,6 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii)
 	return res;
 }
 
-static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter)
-{
-	struct inode *inode = file_inode(iocb->ki_filp);
-
-	return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
-}
-
 /*
  * @return true if an exclusive lock for direct IO writes is needed
  */
@@ -1598,10 +1591,6 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from
 	if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
 		return true;
 
-	/* Parallel dio beyond EOF is not supported, at least for now. */
-	if (fuse_io_past_eof(iocb, from))
-		return true;
-
 	return false;
 }
 
@@ -1620,18 +1609,15 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from,
 		 * New parallal dio allowed only if inode is not in caching
 		 * mode and denies new opens in caching mode. This check
 		 * should be performed only after taking shared inode lock.
-		 * Previous past eof check was without inode lock and might
-		 * have raced, so check it again.
 		 *
-		 * Under the forced-dio latch the cached/uncached accounting is
-		 * bypassed (the latch already guarantees the cache is flushed
-		 * and not repopulated), so just take the shared lock, only
-		 * re-checking the past-eof condition.  The latch is stable
-		 * while the shared lock is held, keeping start/end balanced.
+		 * Under the forced-dio latch the uncached-io accounting is
+		 * bypassed entirely -- fuse_dio_unlock() likewise skips
+		 * fuse_inode_uncached_io_end() -- so do not take a reference
+		 * here.  An unbalanced start would drive fi->iocachectr
+		 * permanently negative and hang the next caching-mode open.
 		 */
-		if (fuse_io_past_eof(iocb, from) ||
-		    (!test_bit(FUSE_I_FORCE_DIO, &fi->state) &&
-		     fuse_inode_uncached_io_start(fi, NULL) != 0)) {
+		if (!test_bit(FUSE_I_FORCE_DIO, &fi->state) &&
+		    fuse_inode_uncached_io_start(fi, NULL) != 0) {
 			inode_unlock_shared(inode);
 			inode_lock(inode);
 			*exclusive = true;
@@ -2148,14 +2134,16 @@ static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
 	return res;
 }
 
-static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
+static ssize_t __fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
+				bool exclusive);
 
 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	ssize_t res;
 
 	if (!is_sync_kiocb(iocb)) {
-		res = fuse_direct_IO(iocb, to);
+		/* exclusive is unused on reads; rollback is write-only */
+		res = __fuse_direct_IO(iocb, to, true);
 	} else {
 		struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
 
@@ -2178,7 +2166,7 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (res > 0) {
 		task_io_account_write(res);
 		if (!is_sync_kiocb(iocb)) {
-			res = fuse_direct_IO(iocb, from);
+			res = __fuse_direct_IO(iocb, from, exclusive);
 		} else {
 			struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
 
@@ -3313,7 +3301,7 @@ static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
 }
 
 static ssize_t
-fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+__fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter, bool exclusive)
 {
 	DECLARE_COMPLETION_ONSTACK(wait);
 	ssize_t ret = 0;
@@ -3407,14 +3395,27 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
 
 	if (iov_iter_rw(iter) == WRITE) {
 		fuse_write_update_attr(inode, pos, ret);
-		/* For extending writes we already hold exclusive lock */
-		if (ret < 0 && offset + count > i_size)
+		/*
+		 * Whole-file rollback is only safe under an exclusive lock.
+		 * Parallel writers commit i_size only on success (nothing to
+		 * undo); the server owns failed-extend cleanup.
+		 */
+		if (exclusive && ret < 0 && offset + count > i_size)
 			fuse_do_truncate(file);
 	}
 
 	return ret;
 }
 
+static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
+{
+	/*
+	 * Only reached via generic_file_direct_write() (caching-mode
+	 * O_DIRECT), which holds the inode lock exclusively.
+	 */
+	return __fuse_direct_IO(iocb, iter, true);
+}
+
 static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
 {
 	int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);