fuse: take inode lock shared for cached writes

Signed-off-by: Horst Birthelmer <hbirthelmer@ddn.com>

Author: hbirth

fs/fuse/file.c

@@ -1768,6 +1768,44 @@ static bool fuse_use_writeback_cache(struct fuse_conn *fc, struct kiocb *iocb,
 	return ret;
 }
 
+/*
+ * @return true if an exclusive lock is needed for a cached (buffered) write.
+ *
+ * Buffered writes normally take the inode lock exclusively, serialising every
+ * writer to the same inode even when they target disjoint regions.  When the
+ * connection uses the DLM (fc->dlm) and writeback caching is in effect for the
+ * write, the server is the authority that serialises writers across the
+ * cluster (the cached data carries a DLM write lock; the synchronous
+ * writethrough path above sends data straight to the server).  The inode rwsem
+ * then no longer has to provide that exclusion - it only needs to keep i_size
+ * and the file-modified bookkeeping stable.  Such writes may therefore share
+ * the inode lock, letting multiple writers to disjoint regions of one file
+ * proceed concurrently (e.g. MPI-IO / IOR shared-file workloads).
+ */
+static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool cache_mode)
+{
+	struct inode *inode = file_inode(iocb->ki_filp);
+	struct fuse_conn *fc = get_fuse_conn(inode);
+
+	/*
+	 * Without the DLM the inode rwsem is the only writer exclusion, and
+	 * outside writeback-cache mode the write is covered by neither the DLM
+	 * write lock nor the synchronous-server path described above.
+	 */
+	if (!fc->dlm || !cache_mode)
+		return true;
+
+	/* O_DIRECT writes fall back to generic_file_direct_write(). */
+	if (iocb->ki_flags & IOCB_DIRECT)
+		return true;
+
+	/* Append needs the eventual EOF - always needs an exclusive lock. */
+	if (iocb->ki_flags & IOCB_APPEND)
+		return true;
+
+	return false;
+}
+
 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from);
 
 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
@@ -1779,7 +1817,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct inode *inode = mapping->host;
 	ssize_t err, count;
 	struct fuse_conn *fc = get_fuse_conn(inode);
+	struct fuse_inode *fi = get_fuse_inode(inode);
 	bool writeback = false;
+	bool cache_mode = false;
+	bool exclusive;
 
 	/*
 	 * The inode may have been latched into forced direct IO after this
@@ -1797,9 +1838,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		if (err)
 			return err;
 
-		if ((!fc->handle_killpriv_v2 ||
-		     !setattr_should_drop_suidgid(idmap, file_inode(file))) &&
-		    fuse_use_writeback_cache(fc, iocb, from)) {
+		cache_mode = !fc->handle_killpriv_v2 ||
+			     !setattr_should_drop_suidgid(idmap, file_inode(file));
+
+		if (cache_mode && fuse_use_writeback_cache(fc, iocb, from)) {
 			writeback = true;
 
 			/*
@@ -1827,7 +1869,11 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		}
 	}
 
-	inode_lock(inode);
+	exclusive = fuse_cache_wr_exclusive_lock(iocb, cache_mode);
+	if (exclusive)
+		inode_lock(inode);
+	else
+		inode_lock_shared(inode);
 
 	err = count = generic_write_checks(iocb, from);
 	if (err <= 0)
@@ -1846,7 +1892,58 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		written = direct_write_fallback(iocb, from, written,
 						fuse_perform_write(iocb, from));
 	} else if (writeback) {
+		loff_t pos = iocb->ki_pos;
+		loff_t end = pos + count;
+		loff_t orig_size = 0;
+		bool extended = false;
+
+		if (fc->dlm && !exclusive && end > i_size_read(inode)) {
+			/*
+			 * Lockless pre-check above keeps in-bounds writes off
+			 * fi->lock; under the shared inode lock i_size only grows
+			 * (extenders take fi->lock, truncate is excluded), so a
+			 * stale read can only over-trigger this slow path, never
+			 * miss an extension.  Re-check authoritatively here.
+			 *
+			 * Pre-growing under fi->lock is required: iomap's own
+			 * i_size update (iomap_write_iter) is a non-atomic
+			 * read-modify-write done under only the folio lock, so
+			 * concurrent shared extenders would lose updates.  Once
+			 * i_size already covers our range iomap's update becomes
+			 * a no-op and all growth stays serialised on fi->lock.
+			 */
+			spin_lock(&fi->lock);
+			orig_size = i_size_read(inode);
+			if (end > orig_size) {
+				i_size_write(inode, end);
+				extended = true;
+			}
+			spin_unlock(&fi->lock);
+
+			/* Zero the tail of the folio straddling the old EOF. */
+			if (extended && orig_size < pos)
+				pagecache_isize_extended(inode, orig_size, pos);
+		}
+
 		written = fuse_writeback_write_iter(iocb, from, file);
+
+		/*
+		 * Reconcile the speculative extension with what was actually
+		 * written.  Only retract the tail if no concurrent extender has
+		 * pushed i_size past our claim; otherwise [reached, end) is a
+		 * legitimate hole inside their extension and must remain.
+		 */
+		if (extended) {
+			loff_t reached = written > 0 ? pos + written : orig_size;
+
+			if (reached < end) {
+				spin_lock(&fi->lock);
+				if (i_size_read(inode) == end)
+					i_size_write(inode, reached);
+				spin_unlock(&fi->lock);
+			}
+		}
+
 		if (written < 0) {
 			err = written;
 			goto out;
@@ -1855,7 +1952,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 		written = fuse_perform_write(iocb, from);
 	}
 out:
-	inode_unlock(inode);
+	if (exclusive)
+		inode_unlock(inode);
+	else
+		inode_unlock_shared(inode);
 	if (written > 0)
 		written = generic_write_sync(iocb, written);