Add reflink copy over SMB3.11 with new FSCTL_DUPLICATE_EXTENTS

PATCH In review Steve French 5 participants 9 messages Jun 29, 2015 3 messages not in the archive

Subsystems (3)

VFS SMB client (CIFS) Kernel SMB3 server (ksmbd)

lore

Share & tools

lore

Steve French

author

2015-06-29 02:21 UTC #0 lore raw

PATCH v2 1/1 Add reflink copy over SMB3.11 with new FSCTL_DUPLICATE_EXTENTS

Update the patch to correct target file size.

Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.

This FSCTL was added in the SMB3.11 dialect (testing was against REFS file system) so have put it as a 3.11 protocol specific operation ("vers=3.1.1" on the mount). Tested at the SMB3 plugfest in Redmond.

It depends on the new FS Attribute (BLOCK_REFCOUNTING) which is used to advertise support for the ability to do this ioctl (if you can support multiple files pointing to the same block than this refcounting ability or equivalent is needed to support the new reflink-like duplicate extent SMB3 ioctl.

Signed-off-by Steve French <[email protected]>

---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)

fs/cifs/cifsglob.h +3 -0

index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
 	int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
 			struct cifsFileInfo *target_file, u64 src_off, u64 len,
 			u64 dest_off);
+	int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+			struct cifsFileInfo *target_file, u64 src_off, u64 len,
+			u64 dest_off);
 	int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
 	ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
 			const unsigned char *, const unsigned char *, char *,

fs/cifs/cifspdu.h +2 -0

index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL	0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING	0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS	0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL	0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID	0x01000000

fs/cifs/ioctl.c +13 -3

index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC	0xCF
 #define CIFS_IOC_COPYCHUNK_FILE	_IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-			unsigned long srcfd, u64 off, u64 len, u64 destoff)
+			unsigned long srcfd, u64 off, u64 len, u64 destoff,
+			bool dup_extents)
 {
 	int rc;
 	struct cifsFileInfo *smb_file_target = dst_file->private_data;

@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
 	truncate_inode_pages_range(&target_inode->i_data, destoff,
 				   PAGE_CACHE_ALIGN(destoff + len)-1);
 
-	if (target_tcon->ses->server->ops->clone_range)
+	if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+		rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+			smb_file_src, smb_file_target, off, len, destoff);
+	else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
 		rc = target_tcon->ses->server->ops->clone_range(xid,
 			smb_file_src, smb_file_target, off, len, destoff);
+	else
+		rc = -EOPNOTSUPP;
 
 	/* force revalidate of size and timestamps of target file now
 	   that target is updated on the server */

@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
 			}
 			break;
 		case CIFS_IOC_COPYCHUNK_FILE:
-			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+			break;
+		case BTRFS_IOC_CLONE:
+			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);
 			break;
 		default:
 			cifs_dbg(FYI, "unsupported ioctl\n");

fs/cifs/smb2ops.c +48 -0

60 lines not on this page · raw message

fs/cifs/smb2pdu.h +8 -0

index 725283a..8e7bbe5 100644
@@ -654,6 +654,14 @@ struct compress_ioctl {
 	__le16 CompressionState; /* See cifspdu.h for possible flag values */
 } __packed;
 
+struct duplicate_extents_to_file {
+	__u64 PersistentFileHandle; /* source file handle, opaque endianness */
+	__u64 VolatileFileHandle;
+	__le64 SourceFileOffset;
+	__le64 TargetFileOffset;
+	__le64 ByteCount;  /* Bytes to be copied */
+} __packed;
+
 struct smb2_ioctl_req {
 	struct smb2_hdr hdr;
 	__le16 StructureSize;	/* Must be 57 */

fs/cifs/smbfsctl.h +1 -0

index 83efa59..d098da6 100644
@@ -78,6 +78,7 @@
 #define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF /* BB add struct */
 #define FSCTL_SET_DEFECT_MANAGEMENT  0x00098134 /* BB add struct */
 #define FSCTL_FILE_LEVEL_TRIM        0x00098208 /* BB add struct */
+#define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344
 #define FSCTL_SIS_LINK_FILES         0x0009C104
 #define FSCTL_PIPE_PEEK              0x0011400C /* BB add struct */
 #define FSCTL_PIPE_TRANSCEIVE        0x0011C017 /* BB add struct */

Signature · 1 line

2.1.4

Darrick J. Wong

2015-06-29 17:27 UTC #1 lore raw

Quoted 90 lines · Steve French

Update the patch to correct target file size.

Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.

Signed-off-by Steve French <[email protected]>

---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)

fs/cifs/cifsglob.h +3 -0

index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
 	int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
 			struct cifsFileInfo *target_file, u64 src_off, u64 len,
 			u64 dest_off);
+	int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+			struct cifsFileInfo *target_file, u64 src_off, u64 len,
+			u64 dest_off);
 	int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
 	ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
 			const unsigned char *, const unsigned char *, char *,

fs/cifs/cifspdu.h +2 -0

index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL	0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING	0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS	0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL	0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID	0x01000000

fs/cifs/ioctl.c +9 -2

index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC	0xCF
 #define CIFS_IOC_COPYCHUNK_FILE	_IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-			unsigned long srcfd, u64 off, u64 len, u64 destoff)
+			unsigned long srcfd, u64 off, u64 len, u64 destoff,
+			bool dup_extents)
 {
 	int rc;
 	struct cifsFileInfo *smb_file_target = dst_file->private_data;

@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
 	truncate_inode_pages_range(&target_inode->i_data, destoff,
 				   PAGE_CACHE_ALIGN(destoff + len)-1);
 
-	if (target_tcon->ses->server->ops->clone_range)
+	if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+		rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+			smb_file_src, smb_file_target, off, len, destoff);
+	else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
 		rc = target_tcon->ses->server->ops->clone_range(xid,
 			smb_file_src, smb_file_target, off, len, destoff);
+	else
+		rc = -EOPNOTSUPP;
 
 	/* force revalidate of size and timestamps of target file now
 	   that target is updated on the server */

fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) · line 218

Quoted 8 lines

fs/cifs/ioctl.c +4 -1

@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
 			}
 			break;
 		case CIFS_IOC_COPYCHUNK_FILE:
-			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+			break;
+		case BTRFS_IOC_CLONE:
+			rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);

Any interest in supporting BTRFS_IOC_CLONE_RANGE or BTRFS_IOC_EXTENT_SAME? It looks like you could easily support the former, and the latter would enable things like duperemove. I've been working on a pile of xfstests to exercise these three ioctls, will post them later today, I hope.

--D

Quoted 107 lines

fs/cifs/ioctl.c +0 -0

 			break;
 		default:
 			cifs_dbg(FYI, "unsupported ioctl\n");

fs/cifs/smb2ops.c +48 -0

index c7d228c..b1e9c0f 100644
@@ -806,6 +806,53 @@ smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
 			    cfile->fid.volatile_fid, cfile->pid, &eof, false);
 }
 
+#ifdef CONFIG_CIFS_SMB311
+static int
+smb2_duplicate_extents(const unsigned int xid,
+			struct cifsFileInfo *srcfile,
+			struct cifsFileInfo *trgtfile, u64 src_off,
+			u64 len, u64 dest_off)
+{
+	int rc;
+	unsigned int ret_data_len;
+	char *retbuf = NULL;
+	struct duplicate_extents_to_file dup_ext_buf;
+	struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink);
+
+	/* server fileays advertise duplicate extent support with this flag */
+	if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) &
+	     FILE_SUPPORTS_BLOCK_REFCOUNTING) == 0)
+		return -EOPNOTSUPP;
+
+	dup_ext_buf.VolatileFileHandle = srcfile->fid.volatile_fid;
+	dup_ext_buf.PersistentFileHandle = srcfile->fid.persistent_fid;
+	dup_ext_buf.SourceFileOffset = cpu_to_le64(src_off);
+	dup_ext_buf.TargetFileOffset = cpu_to_le64(dest_off);
+	dup_ext_buf.ByteCount = cpu_to_le64(len);
+	cifs_dbg(FYI, "duplicate extents: src off %lld dst off %lld len %lld",
+		src_off, dest_off, len);
+
+	rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false);
+	if (rc)
+		goto duplicate_extents_out;
+
+	rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
+			trgtfile->fid.volatile_fid,
+			FSCTL_DUPLICATE_EXTENTS_TO_FILE,
+			true /* is_fsctl */, (char *)&dup_ext_buf,
+			sizeof(struct duplicate_extents_to_file),
+			(char **)&retbuf,
+			&ret_data_len);
+
+	if (ret_data_len > 0)
+		cifs_dbg(FYI, "non-zero response length in duplicate extents");
+
+duplicate_extents_out:
+	return rc;
+}
+#endif /* CONFIG_CIFS_SMB311 */
+
+
 static int
 smb2_set_compression(const unsigned int xid, struct cifs_tcon *tcon,
 		   struct cifsFileInfo *cfile)

@@ -1714,6 +1761,7 @@ struct smb_version_operations smb311_operations = {
 	.create_lease_buf = smb3_create_lease_buf,
 	.parse_lease_buf = smb3_parse_lease_buf,
 	.clone_range = smb2_clone_range,
+	.duplicate_extents = smb2_duplicate_extents,
 /*	.validate_negotiate = smb3_validate_negotiate, */ /* not used in 3.11 */
 	.wp_retry_size = smb2_wp_retry_size,
 	.dir_needs_close = smb2_dir_needs_close,

fs/cifs/smb2pdu.h +8 -0

index 725283a..8e7bbe5 100644
@@ -654,6 +654,14 @@ struct compress_ioctl {
 	__le16 CompressionState; /* See cifspdu.h for possible flag values */
 } __packed;
 
+struct duplicate_extents_to_file {
+	__u64 PersistentFileHandle; /* source file handle, opaque endianness */
+	__u64 VolatileFileHandle;
+	__le64 SourceFileOffset;
+	__le64 TargetFileOffset;
+	__le64 ByteCount;  /* Bytes to be copied */
+} __packed;
+
 struct smb2_ioctl_req {
 	struct smb2_hdr hdr;
 	__le16 StructureSize;	/* Must be 57 */

fs/cifs/smbfsctl.h +1 -0

index 83efa59..d098da6 100644
@@ -78,6 +78,7 @@
 #define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF /* BB add struct */
 #define FSCTL_SET_DEFECT_MANAGEMENT  0x00098134 /* BB add struct */
 #define FSCTL_FILE_LEVEL_TRIM        0x00098208 /* BB add struct */
+#define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344
 #define FSCTL_SIS_LINK_FILES         0x0009C104
 #define FSCTL_PIPE_PEEK              0x0011400C /* BB add struct */
 #define FSCTL_PIPE_TRANSCEIVE        0x0011C017 /* BB add struct */

Signature · 6 lines

2.1.4

--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Steve French

2015-06-29 19:32 UTC #2 lore raw

So far Samba (server) only uses three btrfs ioctls, but that includes clone range:

./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_CLONE_RANGE ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_DESTROY ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_CREATE_V2 ...

But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c

Quoted 238 lines · Darrick J. Wong

Quoted 31 lines · Steve French
Update the patch to correct target file size.
Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.
This FSCTL was added in the SMB3.11 dialect (testing was against REFS file system) so have put it as a 3.11 protocol specific operation ("vers=3.1.1" on the mount). Tested at the SMB3 plugfest in Redmond.
It depends on the new FS Attribute (BLOCK_REFCOUNTING) which is used to advertise support for the ability to do this ioctl (if you can support multiple files pointing to the same block than this refcounting ability or equivalent is needed to support the new reflink-like duplicate extent SMB3 ioctl.
Signed-off-by Steve French <[email protected]>
---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)
fs/cifs/cifsglob.h +0 -0
index a0212ec..81194e6 100644
fs/cifs/cifsglob.h struct smb_version_operations { · line 390
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo
*src_file,
fs/cifs/cifsglob.h struct smb_version_operations { · line 391
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
                      struct cifsFileInfo *target_file, u64 src_off, u64
len,
fs/cifs/cifsglob.h struct smb_version_operations { · line 393
Quoted 2 lines
fs/cifs/cifsglob.h +1 -0
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct cifsFileInfo
*src,
fs/cifs/cifsglob.h struct smb_version_operations { · line 394
Quoted 1 line
fs/cifs/cifsglob.h +1 -0
+                     struct cifsFileInfo *target_file, u64 src_off, u64
len,
fs/cifs/cifsglob.h struct smb_version_operations {
Quoted 4 lines
fs/cifs/cifsglob.h +1 -0
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
                      const unsigned char *, const unsigned char *, char
*,
fs/cifs/cifspdu.h typedef struct { · line 2258
Quoted 9 lines
fs/cifs/cifspdu.h +1 -0
index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse
extend */
fs/cifs/cifspdu.h typedef struct { · line 2259
Quoted 1 line
fs/cifs/cifspdu.h +1 -0
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow ioctl
dup extents */
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid,
Quoted 24 lines
fs/cifs/cifspdu.h +0 -0
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000
fs/cifs/ioctl.c +3 -1
index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-                     unsigned long srcfd, u64 off, u64 len, u64 destoff)
+                     unsigned long srcfd, u64 off, u64 len, u64 destoff,
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;
@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid,
struct file *dst_file,
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
Quoted 5 lines
fs/cifs/ioctl.c +1 -1
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents &&
target_tcon->ses->server->ops->duplicate_extents)
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
Quoted 3 lines
fs/cifs/ioctl.c +3 -0
+             rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+                     smb_file_src, smb_file_target, off, len, destoff);
+     else if (!dup_extents &&
target_tcon->ses->server->ops->clone_range)
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int
Quoted 8 lines
fs/cifs/ioctl.c +2 -0
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len, destoff);
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */
@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int
command, unsigned long arg)
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
Quoted 5 lines
fs/cifs/ioctl.c +1 -1
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0,
false);
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
Quoted 3 lines
fs/cifs/ioctl.c +3 -0
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0,
true);
Any interest in supporting BTRFS_IOC_CLONE_RANGE or BTRFS_IOC_EXTENT_SAME? It looks like you could easily support the former, and the latter would enable things like duperemove. I've been working on a pile of xfstests to exercise these three ioctls, will post them later today, I hope.
--D
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct
Quoted 8 lines
fs/cifs/ioctl.c +0 -0
                      break;
              default:
                      cifs_dbg(FYI, "unsupported ioctl\n");
fs/cifs/smb2ops.c +0 -0
index c7d228c..b1e9c0f 100644
@@ -806,6 +806,53 @@ smb2_set_file_size(const unsigned int xid, struct
cifs_tcon *tcon,
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct · line 806
Quoted 1 line
fs/cifs/smb2ops.c +0 -0
                          cfile->fid.volatile_fid, cfile->pid, &eof,
false);
Quoted 6 lines
fs/cifs/smb2ops.c +4 -0
 }
 
+#ifdef CONFIG_CIFS_SMB311
+static int
+smb2_duplicate_extents(const unsigned int xid,
+                     struct cifsFileInfo *srcfile,
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
Quoted 10 lines
fs/cifs/smb2ops.c +10 -0
+                     struct cifsFileInfo *trgtfile, u64 src_off,
+                     u64 len, u64 dest_off)
+{
+     int rc;
+     unsigned int ret_data_len;
+     char *retbuf = NULL;
+     struct duplicate_extents_to_file dup_ext_buf;
+     struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink);
+
+     /* server fileays advertise duplicate extent support with this
flag */
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
Quoted 10 lines
fs/cifs/smb2ops.c +10 -0
+     if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) &
+          FILE_SUPPORTS_BLOCK_REFCOUNTING) == 0)
+             return -EOPNOTSUPP;
+
+     dup_ext_buf.VolatileFileHandle = srcfile->fid.volatile_fid;
+     dup_ext_buf.PersistentFileHandle = srcfile->fid.persistent_fid;
+     dup_ext_buf.SourceFileOffset = cpu_to_le64(src_off);
+     dup_ext_buf.TargetFileOffset = cpu_to_le64(dest_off);
+     dup_ext_buf.ByteCount = cpu_to_le64(len);
+     cifs_dbg(FYI, "duplicate extents: src off %lld dst off %lld len
%lld",
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
Quoted 3 lines
fs/cifs/smb2ops.c +3 -0
+             src_off, dest_off, len);
+
+     rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len,
false);
fs/cifs/smb2ops.c smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon,
Quoted 13 lines
fs/cifs/smb2ops.c +13 -0
+     if (rc)
+             goto duplicate_extents_out;
+
+     rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid,
+                     trgtfile->fid.volatile_fid,
+                     FSCTL_DUPLICATE_EXTENTS_TO_FILE,
+                     true /* is_fsctl */, (char *)&dup_ext_buf,
+                     sizeof(struct duplicate_extents_to_file),
+                     (char **)&retbuf,
+                     &ret_data_len);
+
+     if (ret_data_len > 0)
+             cifs_dbg(FYI, "non-zero response length in duplicate
extents");
Quoted 11 lines
fs/cifs/smb2ops.c +7 -0
+
+duplicate_extents_out:
+     return rc;
+}
+#endif /* CONFIG_CIFS_SMB311 */
+
+
 static int
 smb2_set_compression(const unsigned int xid, struct cifs_tcon *tcon,
                 struct cifsFileInfo *cfile)
fs/cifs/smb2ops.c struct smb_version_operations smb311_operations = { · line 1765
Quoted 5 lines
fs/cifs/smb2ops.c +1 -0
@@ -1714,6 +1761,7 @@ struct smb_version_operations smb311_operations = {
      .create_lease_buf = smb3_create_lease_buf,
      .parse_lease_buf = smb3_parse_lease_buf,
      .clone_range = smb2_clone_range,
+     .duplicate_extents = smb2_duplicate_extents,
 /*   .validate_negotiate = smb3_validate_negotiate, */ /* not used in
3.11 */
fs/cifs/smb2pdu.h struct compress_ioctl { · line 654
Quoted 8 lines
fs/cifs/smb2ops.c +0 -0
      .wp_retry_size = smb2_wp_retry_size,
      .dir_needs_close = smb2_dir_needs_close,
fs/cifs/smb2pdu.h +0 -0
index 725283a..8e7bbe5 100644
@@ -654,6 +654,14 @@ struct compress_ioctl {
      __le16 CompressionState; /* See cifspdu.h for possible flag values
*/
fs/cifs/smb2pdu.h struct compress_ioctl {
Quoted 4 lines
fs/cifs/smb2pdu.h +2 -0
 } __packed;
 
+struct duplicate_extents_to_file {
+     __u64 PersistentFileHandle; /* source file handle, opaque
endianness */
Quoted 24 lines
fs/cifs/smb2pdu.h +6 -0
+     __u64 VolatileFileHandle;
+     __le64 SourceFileOffset;
+     __le64 TargetFileOffset;
+     __le64 ByteCount;  /* Bytes to be copied */
+} __packed;
+
 struct smb2_ioctl_req {
      struct smb2_hdr hdr;
      __le16 StructureSize;   /* Must be 57 */
fs/cifs/smbfsctl.h +1 -1
index 83efa59..d098da6 100644
@@ -78,6 +78,7 @@
 #define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF /* BB add struct */
 #define FSCTL_SET_DEFECT_MANAGEMENT  0x00098134 /* BB add struct */
 #define FSCTL_FILE_LEVEL_TRIM        0x00098208 /* BB add struct */
+#define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344
 #define FSCTL_SIS_LINK_FILES         0x0009C104
 #define FSCTL_PIPE_PEEK              0x0011400C /* BB add struct */
 #define FSCTL_PIPE_TRANSCEIVE        0x0011C017 /* BB add struct */
--
2.1.4
Quoted 2 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel"
in
Quoted 2 lines
the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 11 lines

*Steve French*

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

[image: cid:E5DD7677-4C3D-4DB7-95DC-31E18398703A]

A message that is not in the archive (sent off-list or not yet received): [email protected]

Steve French

2015-06-29 19:33 UTC #4 lore raw

Quoted 106 lines · Darrick J. Wong

Quoted 90 lines · Steve French

Update the patch to correct target file size.

Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.

Signed-off-by Steve French <steve.french-7I+n7zu2hftEKMMhf/[email protected]>

---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)

fs/cifs/cifsglob.h +3 -0

index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
                      struct cifsFileInfo *target_file, u64 src_off, u64 len,
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+                     struct cifsFileInfo *target_file, u64 src_off, u64 len,
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
                      const unsigned char *, const unsigned char *, char *,

fs/cifs/cifspdu.h +2 -0

index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000

fs/cifs/ioctl.c +9 -2

index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-                     unsigned long srcfd, u64 off, u64 len, u64 destoff)
+                     unsigned long srcfd, u64 off, u64 len, u64 destoff,
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;

@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+             rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+                     smb_file_src, smb_file_target, off, len, destoff);
+     else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len, destoff);
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */

fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) · line 218

Quoted 8 lines

fs/cifs/ioctl.c +4 -1

@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);

Quoted 2 lines

--D

So far Samba (server) only uses three btrfs ioctls, but that includes clone range:

Signature · 9 lines

Steve French

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

A message that is not in the archive (sent off-list or not yet received): CALSSpE5nuE_45PMCKsNBqrg_r9z=+cdCEkBvV1bx=gmiWnRSzw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

Steve French

2015-06-29 19:41 UTC #6 lore raw

Quoted 118 lines · Steve French

Quoted 106 lines · Darrick J. Wong

Quoted 90 lines · Steve French

Update the patch to correct target file size.

Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.

Signed-off-by Steve French <steve.french-7I+n7zu2hftEKMMhf/[email protected]>

---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)

fs/cifs/cifsglob.h +3 -0

index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
                      struct cifsFileInfo *target_file, u64 src_off, u64 len,
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+                     struct cifsFileInfo *target_file, u64 src_off, u64 len,
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
                      const unsigned char *, const unsigned char *, char *,

fs/cifs/cifspdu.h +2 -0

index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000

fs/cifs/ioctl.c +9 -2

index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-                     unsigned long srcfd, u64 off, u64 len, u64 destoff)
+                     unsigned long srcfd, u64 off, u64 len, u64 destoff,
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;

@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+             rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+                     smb_file_src, smb_file_target, off, len, destoff);
+     else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len, destoff);
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */

fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) · line 218

Quoted 8 lines

fs/cifs/ioctl.c +4 -1

@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);

Quoted 2 lines

--D

So far Samba (server) only uses three btrfs ioctls, but that includes clone range:

Quoted 8 lines

But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c

I should also mention that cifs.ko can already do fast(er) server side copy today by using CopyChunk on SMB2/SMB3 but it is emulated in some cases on the Samba server (e.g. when not running on btrfs or with vfs_btrfs enabled) but "cp --reflink" semantics required that it actually do the clone_range/duplicate_extents and Samba can't guarantee that with CopyChunk (FSCTL_SRV_COPYCHUNK_WRITE) - on the other hand the new smb3 network protocol request (SMB3 FSCTL_DUPLICATE_EXTENTS) does guarantee that it is a reflink-like copy.

We need an "smb3cp" utility (perhaps added to cifs-utils) that can try the various types of server side copy and accept parms which allow the user to specify the behavior they want (and the offsets they want) - since few people know about the existing ioctl in cifs.ko which calls CopyChunk (and FSCTL_SRV_COPYCHUNK_WRITE is not as fast as FSCTL_DUPLICATE_EXTENTS)

I was also looking into using DUPLICATE_EXTENTS to enable the two new fallocate features (insert range and collapse range) because it can be called with source and target as the same file.

Signature · 9 lines

Steve French

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

Darrick J. Wong

2015-06-29 21:31 UTC #7 lore raw

In reply to a message not in the archive

Quoted 108 lines · Steve French

Quoted 106 lines · Darrick J. Wong

Quoted 90 lines · Steve French

Update the patch to correct target file size.

Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.

Signed-off-by Steve French <steve.french-7I+n7zu2hftEKMMhf/[email protected]>

---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)

fs/cifs/cifsglob.h +3 -0

index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
                      struct cifsFileInfo *target_file, u64 src_off, u64 len,
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+                     struct cifsFileInfo *target_file, u64 src_off, u64 len,
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
                      const unsigned char *, const unsigned char *, char *,

fs/cifs/cifspdu.h +2 -0

index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000

fs/cifs/ioctl.c +9 -2

index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-                     unsigned long srcfd, u64 off, u64 len, u64 destoff)
+                     unsigned long srcfd, u64 off, u64 len, u64 destoff,
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;

@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+             rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+                     smb_file_src, smb_file_target, off, len, destoff);
+     else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len, destoff);
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */

fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) · line 218

Quoted 8 lines

fs/cifs/ioctl.c +4 -1

@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);

Quoted 5 lines

Quoted 2 lines
--D
So far Samba (server) only uses three btrfs ioctls, but that includes clone range:

Oh, my question was mostly about support for CLONE_RANGE within cifs.ko so that clients could ask for a reflink, which would then be sent off to the server as FSCTL_DUPLICATE_EXTENTS. That doesn't seem too hard, but I didn't see it in ioctl.c and wondered why. :)

Quoted 5 lines

./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_CLONE_RANGE ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_DESTROY ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_CREATE_V2 ...

Quoted 8 lines

But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c

Sounds fairly straightforward on the server side. I'm working on adding the three ioctls I mentioned (CLONE, CLONE_RANGE, EXTENT_SAME) to XFS and others are working on it for ext4; do you think it would be difficult to adapt Samba server to talk to these three btrfs APIs even with a non-btrfs backend?

(Or, maybe someday we might refactor all this into a common syscall or something.)

I guess BTRFS_IOC_EXTENT_SAME would be harder since you'd want to compare file contents on the server, and I'm not sure if SMB can actually do that.

--D

Quoted 17 lines

Signature · 15 lines

-- 

Steve French

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Steve French

2015-06-29 21:49 UTC #8 lore raw

Quoted 165 lines · Darrick J. Wong

Quoted 20 lines · Steve French
Quoted 18 lines · Darrick J. Wong
Quoted 16 lines · Steve French
Update the patch to correct target file size.
Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.
This FSCTL was added in the SMB3.11 dialect (testing was against REFS file system) so have put it as a 3.11 protocol specific operation ("vers=3.1.1" on the mount). Tested at the SMB3 plugfest in Redmond.
It depends on the new FS Attribute (BLOCK_REFCOUNTING) which is used to advertise support for the ability to do this ioctl (if you can support multiple files pointing to the same block than this refcounting ability or equivalent is needed to support the new reflink-like duplicate extent SMB3 ioctl.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Signed-off-by Steve French <[email protected]>
---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48
++++++++++++++++++++++++++++++++++++++++++++++++
Quoted 10 lines
Quoted 10 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 390
Quoted 10 lines
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)
fs/cifs/cifsglob.h +0 -0
index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo
*src_file,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 391
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
                      struct cifsFileInfo *target_file, u64 src_off,
u64 len,
Quoted 2 lines
Quoted 2 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 393
Quoted 2 lines
fs/cifs/cifsglob.h +1 -0
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct
cifsFileInfo *src,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 394
Quoted 1 line
fs/cifs/cifsglob.h +1 -0
+                     struct cifsFileInfo *target_file, u64 src_off,
u64 len,
Quoted 2 lines
Quoted 2 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 396
Quoted 2 lines
fs/cifs/cifsglob.h +1 -0
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon
*);
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 397
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon
*,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 398
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
                      const unsigned char *, const unsigned char *,
char *,
Quoted 9 lines
Quoted 9 lines
fs/cifs/cifspdu.h typedef struct { · line 2258
Quoted 9 lines
fs/cifs/cifspdu.h +1 -0
index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse
extend */
Quoted 1 line
Quoted 1 line
fs/cifs/cifspdu.h typedef struct { · line 2259
Quoted 1 line
fs/cifs/cifspdu.h +1 -0
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow
ioctl dup extents */
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
fs/cifs/cifspdu.h +0 -0
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000
fs/cifs/ioctl.c +1 -0
index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
Quoted 4 lines
Quoted 4 lines
fs/cifs/ioctl.c
Quoted 4 lines
fs/cifs/ioctl.c +0 -0
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file
*dst_file,
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c line 39
Quoted 1 line
fs/cifs/ioctl.c +0 -1
-                     unsigned long srcfd, u64 off, u64 len, u64
destoff)
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c line 40
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     unsigned long srcfd, u64 off, u64 len, u64
destoff,
Quoted 5 lines
Quoted 5 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid,
Quoted 5 lines
fs/cifs/ioctl.c +1 -0
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;
@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid,
struct file *dst_file,
Quoted 5 lines
Quoted 5 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
Quoted 5 lines
fs/cifs/ioctl.c +1 -1
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents &&
target_tcon->ses->server->ops->duplicate_extents)
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 115
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+             rc =
target_tcon->ses->server->ops->duplicate_extents(xid,
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 116
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     smb_file_src, smb_file_target, off, len,
destoff);
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 117
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+     else if (!dup_extents &&
target_tcon->ses->server->ops->clone_range)
Quoted 2 lines
Quoted 2 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 119
Quoted 2 lines
fs/cifs/ioctl.c +0 -0
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len,
destoff);
Quoted 6 lines
Quoted 6 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned
Quoted 6 lines
fs/cifs/ioctl.c +2 -0
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */
@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned
int command, unsigned long arg)
Quoted 4 lines
Quoted 4 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
Quoted 4 lines
fs/cifs/ioctl.c +0 -1
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0);
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned · line 215
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0, false);
Quoted 3 lines
Quoted 3 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
Quoted 3 lines
fs/cifs/ioctl.c +3 -0
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0, true);
Quoted 2 lines
Quoted 2 lines
Any interest in supporting BTRFS_IOC_CLONE_RANGE or
BTRFS_IOC_EXTENT_SAME?
Quoted 2 lines
Quoted 2 lines
It looks like you could easily support the former, and the latter would enable things like duperemove. I've been working on a pile of
xfstests to
Quoted 7 lines
Quoted 4 lines
exercise these three ioctls, will post them later today, I hope.
--D
So far Samba (server) only uses three btrfs ioctls, but that includes clone range:
Oh, my question was mostly about support for CLONE_RANGE within cifs.ko so that clients could ask for a reflink, which would then be sent off to the server as FSCTL_DUPLICATE_EXTENTS. That doesn't seem too hard, but I didn't see it in ioctl.c and wondered why. :)
Quoted 5 lines
./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_CLONE_RANGE ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_DESTROY ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_CREATE_V2 ...
Quoted 8 lines
But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c
Sounds fairly straightforward on the server side. I'm working on adding the three ioctls I mentioned (CLONE, CLONE_RANGE, EXTENT_SAME) to XFS and others are working on it for ext4; do you think it would be difficult to adapt Samba server to talk to these three btrfs APIs even with a non-btrfs backend?

Quoted 7 lines

(Or, maybe someday we might refactor all this into a common syscall or something.)
I guess BTRFS_IOC_EXTENT_SAME would be harder since you'd want to compare file contents on the server, and I'm not sure if SMB can actually do that.

cifs.ko with the recent patches, does support IOC_CLONE (ie
BTRFS_IOC_CLONE) but I didn't add BTRFS_CLONE_RANGE which would be easy enough to add to cifs.ko (although would be a little better, and perhaps less confusing, if there were a system ioctl definition that was an exact equivalent of BTRFS_IOC_CLONE and BTRFS_IOC_CLONE_RANGE but didn't use the name BTRFS, but last I checked the cp command it only knows about BTRFS_IOC_CLONE so we are stuck using that ioctl number).

If you call CIFS_IOC_COPYCHUNK_FILE ioctl today in cifs.ko (even without the patches), for SMB3 mounts it will send a FSCTL_SRV_COPYCHUNK_WRITE SMB3 request to the server which Samba will either emulate in user space as a somewhat faster file copy, or pass directly to btrfs if vfs_btrfs is loaded and the server will do a BTRFS_IOC_CLONE_RANGE.

With the recent patch to cifs.ko it will be easier since for cifs --reflink we will only issue DUPLICATE_EXTENTS IOCTL over the wire (which Samba can convert to BRTFS_IOC_CLONE or BTRFS_IOC_CLONE_RANGE) if the server advertises the file system support for the corresponding attribute flag.

Signature · 11 lines

*Steve French*

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

[image: cid:E5DD7677-4C3D-4DB7-95DC-31E18398703A]

Steve French

2015-06-29 21:49 UTC #9 lore raw

PATCH v2 1/1 Add reflink copy over SMB3.11 with new FSCTL_DUPLICATE_EXTENTS

Quoted 165 lines · Darrick J. Wong

Quoted 20 lines · Steve French
Quoted 18 lines · Darrick J. Wong
Quoted 16 lines · Steve French
Update the patch to correct target file size.
Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.
This FSCTL was added in the SMB3.11 dialect (testing was against REFS file system) so have put it as a 3.11 protocol specific operation ("vers=3.1.1" on the mount). Tested at the SMB3 plugfest in Redmond.
It depends on the new FS Attribute (BLOCK_REFCOUNTING) which is used to advertise support for the ability to do this ioctl (if you can support multiple files pointing to the same block than this refcounting ability or equivalent is needed to support the new reflink-like duplicate extent SMB3 ioctl.
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Signed-off-by Steve French <[email protected]>
---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48
++++++++++++++++++++++++++++++++++++++++++++++++
Quoted 10 lines
Quoted 10 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 390
Quoted 10 lines
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)
fs/cifs/cifsglob.h +0 -0
index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo
*src_file,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 391
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
                      struct cifsFileInfo *target_file, u64 src_off,
u64 len,
Quoted 2 lines
Quoted 2 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 393
Quoted 2 lines
fs/cifs/cifsglob.h +1 -0
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct
cifsFileInfo *src,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 394
Quoted 1 line
fs/cifs/cifsglob.h +1 -0
+                     struct cifsFileInfo *target_file, u64 src_off,
u64 len,
Quoted 2 lines
Quoted 2 lines
fs/cifs/cifsglob.h struct smb_version_operations { · line 396
Quoted 2 lines
fs/cifs/cifsglob.h +1 -0
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon
*);
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 397
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon
*,
Quoted 1 line
Quoted 1 line
fs/cifs/cifsglob.h struct smb_version_operations { · line 398
Quoted 1 line
fs/cifs/cifsglob.h +0 -0
                      const unsigned char *, const unsigned char *,
char *,
Quoted 9 lines
Quoted 9 lines
fs/cifs/cifspdu.h typedef struct { · line 2258
Quoted 9 lines
fs/cifs/cifspdu.h +1 -0
index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse
extend */
Quoted 1 line
Quoted 1 line
fs/cifs/cifspdu.h typedef struct { · line 2259
Quoted 1 line
fs/cifs/cifspdu.h +1 -0
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow
ioctl dup extents */
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
fs/cifs/cifspdu.h +0 -0
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000
fs/cifs/ioctl.c +1 -0
index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
Quoted 4 lines
Quoted 4 lines
fs/cifs/ioctl.c line 39
Quoted 4 lines
fs/cifs/ioctl.c +0 -0
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file
*dst_file,
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c line 39
Quoted 1 line
fs/cifs/ioctl.c +0 -1
-                     unsigned long srcfd, u64 off, u64 len, u64
destoff)
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c line 40
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     unsigned long srcfd, u64 off, u64 len, u64
destoff,
Quoted 5 lines
Quoted 5 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid,
Quoted 5 lines
fs/cifs/ioctl.c +1 -0
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;
@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid,
struct file *dst_file,
Quoted 5 lines
Quoted 5 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 114
Quoted 5 lines
fs/cifs/ioctl.c +1 -1
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents &&
target_tcon->ses->server->ops->duplicate_extents)
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 115
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+             rc =
target_tcon->ses->server->ops->duplicate_extents(xid,
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 116
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     smb_file_src, smb_file_target, off, len,
destoff);
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 117
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+     else if (!dup_extents &&
target_tcon->ses->server->ops->clone_range)
Quoted 2 lines
Quoted 2 lines
fs/cifs/ioctl.c static long cifs_ioctl_clone(unsigned int xid, · line 119
Quoted 2 lines
fs/cifs/ioctl.c +0 -0
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len,
destoff);
Quoted 6 lines
Quoted 6 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned
Quoted 6 lines
fs/cifs/ioctl.c +2 -0
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */
@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned
int command, unsigned long arg)
Quoted 4 lines
Quoted 4 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned · line 208
Quoted 4 lines
fs/cifs/ioctl.c +0 -1
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0);
Quoted 1 line
Quoted 1 line
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned · line 215
Quoted 1 line
fs/cifs/ioctl.c +1 -0
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0, false);
Quoted 3 lines
Quoted 3 lines
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned · line 218
Quoted 3 lines
fs/cifs/ioctl.c +3 -0
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0,
0, true);
Quoted 2 lines
Quoted 2 lines
Any interest in supporting BTRFS_IOC_CLONE_RANGE or
BTRFS_IOC_EXTENT_SAME?
Quoted 2 lines
Quoted 2 lines
It looks like you could easily support the former, and the latter would enable things like duperemove. I've been working on a pile of
xfstests to
Quoted 7 lines
Quoted 4 lines
exercise these three ioctls, will post them later today, I hope.
--D
So far Samba (server) only uses three btrfs ioctls, but that includes clone range:
Oh, my question was mostly about support for CLONE_RANGE within cifs.ko so that clients could ask for a reflink, which would then be sent off to the server as FSCTL_DUPLICATE_EXTENTS. That doesn't seem too hard, but I didn't see it in ioctl.c and wondered why. :)
Quoted 5 lines
./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_CLONE_RANGE ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_DESTROY ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_CREATE_V2 ...
Quoted 8 lines
But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c
Sounds fairly straightforward on the server side. I'm working on adding the three ioctls I mentioned (CLONE, CLONE_RANGE, EXTENT_SAME) to XFS and others are working on it for ext4; do you think it would be difficult to adapt Samba server to talk to these three btrfs APIs even with a non-btrfs backend?

Quoted 7 lines

(Or, maybe someday we might refactor all this into a common syscall or something.)
I guess BTRFS_IOC_EXTENT_SAME would be harder since you'd want to compare file contents on the server, and I'm not sure if SMB can actually do that.

Signature · 28 lines

*Steve French*

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

[image: cid:E5DD7677-4C3D-4DB7-95DC-31E18398703A]




-- 

*Steve French*

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com

[image: cid:E5DD7677-4C3D-4DB7-95DC-31E18398703A]

A message that is not in the archive (sent off-list or not yet received): CALSSpE6YTLmB6zQ6foRYXOMH+SyhhHGEvSNPo0iMqQWTs9hbDw-JsoAwUIsXosN+BqQ9rBEUg@public.gmane.org

Steve French

2015-06-29 21:50 UTC #11 lore raw

PATCH v2 1/1 Add reflink copy over SMB3.11 with new FSCTL_DUPLICATE_EXTENTS

Quoted 141 lines · Darrick J. Wong

Quoted 108 lines · Steve French
Quoted 106 lines · Darrick J. Wong
Quoted 90 lines · Steve French
Update the patch to correct target file size.
Getting fantastic copy performance with cp --reflink over SMB3.11 using the new FSCTL_DUPLICATE_EXTENTS.
This FSCTL was added in the SMB3.11 dialect (testing was against REFS file system) so have put it as a 3.11 protocol specific operation ("vers=3.1.1" on the mount). Tested at the SMB3 plugfest in Redmond.
It depends on the new FS Attribute (BLOCK_REFCOUNTING) which is used to advertise support for the ability to do this ioctl (if you can support multiple files pointing to the same block than this refcounting ability or equivalent is needed to support the new reflink-like duplicate extent SMB3 ioctl.
Signed-off-by Steve French <steve.french-7I+n7zu2hftEKMMhf/[email protected]>
---
 fs/cifs/cifsglob.h |  3 +++
 fs/cifs/cifspdu.h  |  2 ++
 fs/cifs/ioctl.c    | 16 +++++++++++++---
 fs/cifs/smb2ops.c  | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/cifs/smb2pdu.h  |  8 ++++++++
 fs/cifs/smbfsctl.h |  1 +
 6 files changed, 75 insertions(+), 3 deletions(-)
fs/cifs/cifsglob.h +3 -0
index a0212ec..81194e6 100644
@@ -390,6 +390,9 @@ struct smb_version_operations {
      int (*clone_range)(const unsigned int, struct cifsFileInfo *src_file,
                      struct cifsFileInfo *target_file, u64 src_off, u64 len,
                      u64 dest_off);
+     int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src,
+                     struct cifsFileInfo *target_file, u64 src_off, u64 len,
+                     u64 dest_off);
      int (*validate_negotiate)(const unsigned int, struct cifs_tcon *);
      ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *,
                      const unsigned char *, const unsigned char *, char *,
fs/cifs/cifspdu.h +2 -0
index 998a66f..47b030d 100644
@@ -2255,6 +2255,8 @@ typedef struct {
 
 
 /* List of FileSystemAttributes - see 2.5.1 of MS-FSCC */
+#define FILE_SUPPORTS_SPARSE_VDL     0x10000000 /* faster nonsparse extend */
+#define FILE_SUPPORTS_BLOCK_REFCOUNTING      0x08000000 /* allow ioctl dup extents */
 #define FILE_SUPPORT_INTEGRITY_STREAMS       0x04000000
 #define FILE_SUPPORTS_USN_JOURNAL    0x02000000
 #define FILE_SUPPORTS_OPEN_BY_FILE_ID        0x01000000
fs/cifs/ioctl.c +9 -2
index 8b7898b..7843b19 100644
@@ -31,12 +31,14 @@
 #include "cifsproto.h"
 #include "cifs_debug.h"
 #include "cifsfs.h"
+#include <linux/btrfs.h>
 
 #define CIFS_IOCTL_MAGIC     0xCF
 #define CIFS_IOC_COPYCHUNK_FILE      _IOW(CIFS_IOCTL_MAGIC, 3, int)
 
 static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
-                     unsigned long srcfd, u64 off, u64 len, u64 destoff)
+                     unsigned long srcfd, u64 off, u64 len, u64 destoff,
+                     bool dup_extents)
 {
      int rc;
      struct cifsFileInfo *smb_file_target = dst_file->private_data;
@@ -109,9 +111,14 @@ static long cifs_ioctl_clone(unsigned int xid, struct file *dst_file,
      truncate_inode_pages_range(&target_inode->i_data, destoff,
                                 PAGE_CACHE_ALIGN(destoff + len)-1);
 
-     if (target_tcon->ses->server->ops->clone_range)
+     if (dup_extents && target_tcon->ses->server->ops->duplicate_extents)
+             rc = target_tcon->ses->server->ops->duplicate_extents(xid,
+                     smb_file_src, smb_file_target, off, len, destoff);
+     else if (!dup_extents && target_tcon->ses->server->ops->clone_range)
              rc = target_tcon->ses->server->ops->clone_range(xid,
                      smb_file_src, smb_file_target, off, len, destoff);
+     else
+             rc = -EOPNOTSUPP;
 
      /* force revalidate of size and timestamps of target file now
         that target is updated on the server */
fs/cifs/ioctl.c long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) · line 218
Quoted 8 lines
fs/cifs/ioctl.c +4 -1
@@ -205,7 +212,10 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg)
                      }
                      break;
              case CIFS_IOC_COPYCHUNK_FILE:
-                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0);
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, false);
+                     break;
+             case BTRFS_IOC_CLONE:
+                     rc = cifs_ioctl_clone(xid, filep, arg, 0, 0, 0, true);
Any interest in supporting BTRFS_IOC_CLONE_RANGE or BTRFS_IOC_EXTENT_SAME? It looks like you could easily support the former, and the latter would enable things like duperemove. I've been working on a pile of xfstests to exercise these three ioctls, will post them later today, I hope.
Quoted 5 lines
Quoted 2 lines
--D
So far Samba (server) only uses three btrfs ioctls, but that includes clone range:
Oh, my question was mostly about support for CLONE_RANGE within cifs.ko so that clients could ask for a reflink, which would then be sent off to the server as FSCTL_DUPLICATE_EXTENTS. That doesn't seem too hard, but I didn't see it in ioctl.c and wondered why. :)
Quoted 5 lines
./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_CLONE_RANGE ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_DESTROY ... ./source3/modules/vfs_btrfs.c:#define BTRFS_IOC_SNAP_CREATE_V2 ...
Quoted 8 lines
But btrfs clone range is only (optionally if present) used to handle the "CopyChunk" API (SMB3 and CIFS network protocol requests), and Samba server doesn't have code for the FSCTL_DUPLICATE_EXTENTS fsctl (SMB3 network protocol request) yet. It looks trivial to map that to
BTRFS_IOC_CLONE_RANGE in the Samba server. Would need to add trivial
parsing in smbd/smb2_ioctl_network_fs.c and a new VFS entry point in the Samba VFS and a small helper routine in source3/modules/vfs_btrfs.c
Sounds fairly straightforward on the server side. I'm working on adding the three ioctls I mentioned (CLONE, CLONE_RANGE, EXTENT_SAME) to XFS and others are working on it for ext4; do you think it would be difficult to adapt Samba server to talk to these three btrfs APIs even with a non-btrfs backend?

Quoted 6 lines

(Or, maybe someday we might refactor all this into a common syscall or something.)
I guess BTRFS_IOC_EXTENT_SAME would be harder since you'd want to compare file contents on the server, and I'm not sure if SMB can actually do that.

Signature · 25 lines

Steve French

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com





-- 

Steve French

Principal Systems Engineer: Protocols

W: 512-918-9276

C:  512-501-9669

www.primarydata.com