fscrypt: cache decrypted symlink target in ->i_link
Eric Biggers
maintainerPATCH 1/1 fscrypt: cache decrypted symlink target in ->i_link
From: Eric Biggers <[email protected]>
Path lookups that traverse encrypted symlink(s) are very slow because each encrypted symlink needs to be decrypted each time it's followed.
Make encrypted symlinks faster by caching the decrypted symlink target in ->i_link. The first call to ->get_link() sets it; later calls simply return it. ->symlink() also sets it when the symlink is created.
When the inode's ->i_crypt_info is freed, ->i_link is freed too.
Note: RCU-delayed freeing of ->i_link is not yet implemented. Therefore, for now even when ->i_link is set, path lookups must continue to drop out of RCU-walk mode when following an encrypted symlink.
--- fs/crypto/hooks.c | 39 ++++++++++++++++++++++++++++++++------- fs/crypto/keyinfo.c | 4 ++++ fs/ext4/symlink.c | 4 ++++ fs/f2fs/namei.c | 4 ++++ include/linux/fscrypt.h | 23 +++++++++++++++++++++++ 5 files changed, 67 insertions(+), 7 deletions(-)
fs/crypto/hooks.c +32 -7
@@ -179,11 +179,9 @@ int __fscrypt_encrypt_symlink(struct inode *inode, const char *target, sd->len =cpu_to_le16 (ciphertext_len); err =fname_encrypt (inode, &iname, sd->encrypted_path, ciphertext_len); -if (err) { -if (!disk_link->name) -kfree (sd); -return err; - } +if (err) + goto err_free_sd; +/* * Null-terminating the ciphertext doesn't make sense, but we still * count the null terminator in the length, so we might as well
@@ -191,9 +189,20 @@ int __fscrypt_encrypt_symlink(struct inode *inode, const char *target,*/ sd->encrypted_path[ciphertext_len] ='\0' ; +/* Cache the plaintext symlink target for later use by ->get_link(). */ + err = -ENOMEM ; + inode->i_link =kmemdup (target, len +1 ,GFP_NOFS ); +if (!inode->i_link) + goto err_free_sd; +if (!disk_link->name) disk_link->name = (unsigned char *)sd;return 0 ; + +err_free_sd: +if (!disk_link->name) +kfree (sd); +return err; }EXPORT_SYMBOL_GPL (__fscrypt_encrypt_symlink);
@@ -202,7 +211,7 @@ EXPORT_SYMBOL_GPL(__fscrypt_encrypt_symlink);* @inode: the symlink inode * @caddr: the on-disk contents of the symlink * @max_size: size of @caddr buffer -* @done: if successful, will be set up to free the returned target +* @done: if successful, will be set up to free the returned target if needed * * If the symlink's encryption key is available, we decrypt its target. * Otherwise, we encode its target for presentation.
@@ -217,12 +226,18 @@ const char *fscrypt_get_symlink(struct inode *inode, const void *caddr, {const struct fscrypt_symlink_data *sd;struct fscrypt_str cstr, pstr; +bool has_key;int err;/* This is for encrypted symlinks only */ if (WARN_ON (!IS_ENCRYPTED (inode)))return ERR_PTR (-EINVAL ); +/* If the decrypted target is already cached, just return it. */ + pstr.name = (char *)fscrypt_get_cached_symlink (inode); +if (pstr.name) +return pstr.name; +/* * Try to set up the symlink's encryption key, but we can continue * regardless of whether the key is available or not.
@@ -230,6 +245,7 @@ const char *fscrypt_get_symlink(struct inode *inode, const void *caddr, err =fscrypt_get_encryption_info (inode);if (err)return ERR_PTR (err); + has_key =fscrypt_has_encryption_key (inode);/* * For historical reasons, encrypted symlink targets are prefixed with
@@ -261,7 +277,16 @@ const char *fscrypt_get_symlink(struct inode *inode, const void *caddr, goto err_kfree; pstr.name[pstr.len] ='\0' ; -set_delayed_call (done, kfree_link, pstr.name); + +/* +* Cache decrypted symlink targets in i_link for later use. Don't cache +* symlink targets encoded without the key, since those become outdated +* once the key is added. This pairs with fscrypt_get_cached_symlink(). +*/ +if (!has_key || +cmpxchg_release (&inode->i_link,NULL , pstr.name) !=NULL ) +set_delayed_call (done, kfree_link, pstr.name); +return pstr.name; err_kfree:
fs/crypto/keyinfo.c +4 -0
@@ -586,6 +586,10 @@ EXPORT_SYMBOL(fscrypt_get_encryption_info);void fscrypt_put_encryption_info (struct inode *inode) { +if (IS_ENCRYPTED (inode) &&S_ISLNK (inode->i_mode)) { +kfree (inode->i_link); + inode->i_link =NULL ; + }put_crypt_info (inode->i_crypt_info); inode->i_crypt_info =NULL ; }
fs/ext4/symlink.c +4 -0
@@ -35,6 +35,10 @@ static const char *ext4_encrypted_get_link(struct dentry *dentry,if (!dentry)return ERR_PTR (-ECHILD ); + paddr =fscrypt_get_cached_symlink (inode); +if (paddr) +return paddr; +if (ext4_inode_is_fast_symlink (inode)) { caddr =EXT4_I (inode)->i_data; max_size =sizeof (EXT4_I (inode)->i_data);
fs/f2fs/namei.c +4 -0
@@ -1212,6 +1212,10 @@ static const char *f2fs_encrypted_get_link(struct dentry *dentry,if (!dentry)return ERR_PTR (-ECHILD ); + target =fscrypt_get_cached_symlink (inode); +if (target) +return target; + page =read_mapping_page (inode->i_mapping,0 ,NULL );if (IS_ERR (page))return ERR_CAST (page);
include/linux/fscrypt.h +23 -0
@@ -231,6 +231,24 @@ extern int __fscrypt_encrypt_symlink(struct inode *inode, const char *target,extern const char *fscrypt_get_symlink (struct inode *inode,const void *caddr,unsigned int max_size,struct delayed_call *done); +/** +* fscrypt_get_cached_symlink - get the cached decrypted symlink target +* @inode: inode of an encrypted symlink +* +* If the decrypted symlink target was already cached, return it. +* +* This must not be called in RCU-walk mode, since currently the cached symlink +* target is freed without waiting for an RCU grace period. +* +* Return: the symlink target if cached; NULL if uncached; +* or ERR_PTR(-EOPNOTSUPP) when !CONFIG_FS_ENCRYPTION. +*/ +static inline const char *fscrypt_get_cached_symlink (const struct inode *inode) +{ +/* Pairs with the cmpxchg_release() in fscrypt_get_symlink() */ +return smp_load_acquire (&inode->i_link); +} +#else /* !CONFIG_FS_ENCRYPTION */ static inline bool fscrypt_has_encryption_key (const struct inode *inode)
@@ -446,6 +464,11 @@ static inline const char *fscrypt_get_symlink(struct inode *inode, {return ERR_PTR (-EOPNOTSUPP ); } + +static inline const char *fscrypt_get_cached_symlink (const struct inode *inode) +{ +return ERR_PTR (-EOPNOTSUPP ); +}#endif /* !CONFIG_FS_ENCRYPTION */ /**
Signature · 8 lines
2.21.0.392.gf8f6787159e-goog _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Al Viro
maintainerQuoted 9 lines · Eric Biggers
From: Eric Biggers <[email protected]>
Path lookups that traverse encrypted symlink(s) are very slow because each encrypted symlink needs to be decrypted each time it's followed.
Make encrypted symlinks faster by caching the decrypted symlink target in ->i_link. The first call to ->get_link() sets it; later calls simply return it. ->symlink() also sets it when the symlink is created.
Quoted 5 lines
When the inode's ->i_crypt_info is freed, ->i_link is freed too.
Note: RCU-delayed freeing of ->i_link is not yet implemented. Therefore, for now even when ->i_link is set, path lookups must continue to drop out of RCU-walk mode when following an encrypted symlink.
And how the devil would they continue to do that, if I might ask? ->get_link() is *NOT* called if ->i_link is non-NULL, period.
Eric Biggers
maintainerQuoted 10 lines · Al Viro
Quoted 9 lines · Eric Biggers
From: Eric Biggers <[email protected]>
Path lookups that traverse encrypted symlink(s) are very slow because each encrypted symlink needs to be decrypted each time it's followed.
Make encrypted symlinks faster by caching the decrypted symlink target in ->i_link. The first call to ->get_link() sets it; later calls simply return it. ->symlink() also sets it when the symlink is created.
Quoted 8 lines
Quoted 5 lines
When the inode's ->i_crypt_info is freed, ->i_link is freed too.
Note: RCU-delayed freeing of ->i_link is not yet implemented. Therefore, for now even when ->i_link is set, path lookups must continue to drop out of RCU-walk mode when following an encrypted symlink.
And how the devil would they continue to do that, if I might ask? ->get_link() is *NOT* called if ->i_link is non-NULL, period.
You're right, I didn't notice that ->get_link() isn't called when ->i_link is non-NULL.
But that being the case, what's the point of simple_get_link()?
- Eric
Al Viro
maintainerQuoted 17 lines · Eric Biggers
Quoted 16 lines · Al Viro
Quoted 14 lines · Eric Biggers
From: Eric Biggers <[email protected]>
Path lookups that traverse encrypted symlink(s) are very slow because each encrypted symlink needs to be decrypted each time it's followed.
Make encrypted symlinks faster by caching the decrypted symlink target in ->i_link. The first call to ->get_link() sets it; later calls simply return it. ->symlink() also sets it when the symlink is created.
When the inode's ->i_crypt_info is freed, ->i_link is freed too.
Note: RCU-delayed freeing of ->i_link is not yet implemented. Therefore, for now even when ->i_link is set, path lookups must continue to drop out of RCU-walk mode when following an encrypted symlink.
Quoted 7 lines
Quoted 2 lines
And how the devil would they continue to do that, if I might ask? ->get_link() is *NOT* called if ->i_link is non-NULL, period.
You're right, I didn't notice that ->get_link() isn't called when ->i_link is non-NULL.
But that being the case, what's the point of
simple_get_link()?
Non-NULL ->get_link() => DCACHE_SYMLINK_TYPE in ->d_flags => d_is_symlink() true => step_into() progresses to pick_link().
IOW, non-NULL ->get_link() is what tells you that we have a symlink there.
Eric Biggers
maintainerQuoted 24 lines · Al Viro
Quoted 23 lines · Eric Biggers
Quoted 10 lines · Al Viro
Quoted 9 lines · Eric Biggers
From: Eric Biggers <[email protected]>
Path lookups that traverse encrypted symlink(s) are very slow because each encrypted symlink needs to be decrypted each time it's followed.
Make encrypted symlinks faster by caching the decrypted symlink target in ->i_link. The first call to ->get_link() sets it; later calls simply return it. ->symlink() also sets it when the symlink is created.
Quoted 8 lines
Quoted 5 lines
When the inode's ->i_crypt_info is freed, ->i_link is freed too.
Note: RCU-delayed freeing of ->i_link is not yet implemented. Therefore, for now even when ->i_link is set, path lookups must continue to drop out of RCU-walk mode when following an encrypted symlink.
And how the devil would they continue to do that, if I might ask? ->get_link() is *NOT* called if ->i_link is non-NULL, period.
You're right, I didn't notice that ->get_link() isn't called when ->i_link is non-NULL.
Quoted 7 lines
Quoted 1 line
But that being the case, what's the point of
simple_get_link()?Non-NULL ->get_link() =>
DCACHE_SYMLINK_TYPEin ->d_flags =>d_is_symlink()true =>step_into()progresses topick_link().IOW, non-NULL ->get_link() is what tells you that we have a symlink there.
I think that's pretty unintuitive. The fact that multiple filesystems including ext4 set ->i_link on fast symlinks, then set ->get_link() to a function that returns ->i_link, made me assume that's the mechanism by which such symlink targets are returned to the VFS. When in fact fs/namei.c just uses ->i_link, and never calls ->get_link().
Is there any reason why d_flags_for_inode() doesn't check S_ISLNK() instead, and then fs/namei.c would call ->get_link() if non-NULL, otherwise use ->i_link?
- Eric
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Al Viro
maintainerQuoted 6 lines · Eric Biggers
Quoted 5 lines
Non-NULL ->get_link() =>
DCACHE_SYMLINK_TYPEin ->d_flags =>d_is_symlink()true =>step_into()progresses topick_link().IOW, non-NULL ->get_link() is what tells you that we have a symlink there.
Quoted 8 lines
I think that's pretty unintuitive. The fact that multiple filesystems including ext4 set ->i_link on fast symlinks, then set ->get_link() to a function that returns ->i_link, made me assume that's the mechanism by which such symlink targets are returned to the VFS. When in fact
fs/namei.cjust uses ->i_link, and never calls ->get_link().Is there any reason why
d_flags_for_inode()doesn't checkS_ISLNK()instead, and thenfs/namei.cwould call ->get_link() if non-NULL, otherwise use ->i_link?
Extra check and dereference on hot path with no visible benefits of doing it that way, for starters. Really, what _is_ the benefit of pessimizing that? Most of the symlinks we run into will have ->i_link set; checking ->i_op->get_link first is extra work for no good reason...
What's more, ->get_link is visible in inode_operations; ->i_link (let alone ->i_mode) isn't. As it is, we can easily tell symlink inode_operations from everything else on the source level. With your scheme we won't.
Eric Biggers
maintainerQuoted 17 lines · Al Viro
Quoted 6 lines · Eric Biggers
Quoted 5 lines
Non-NULL ->get_link() =>
DCACHE_SYMLINK_TYPEin ->d_flags =>d_is_symlink()true =>step_into()progresses topick_link().IOW, non-NULL ->get_link() is what tells you that we have a symlink there.
Quoted 8 lines
I think that's pretty unintuitive. The fact that multiple filesystems including ext4 set ->i_link on fast symlinks, then set ->get_link() to a function that returns ->i_link, made me assume that's the mechanism by which such symlink targets are returned to the VFS. When in fact
fs/namei.cjust uses ->i_link, and never calls ->get_link().Is there any reason why
d_flags_for_inode()doesn't checkS_ISLNK()instead, and thenfs/namei.cwould call ->get_link() if non-NULL, otherwise use ->i_link?
Quoted 8 lines
Extra check and dereference on hot path with no visible benefits of doing it that way, for starters. Really, what _is_ the benefit of pessimizing that? Most of the symlinks we run into will have ->i_link set; checking ->i_op->get_link first is extra work for no good reason...
What's more, ->get_link is visible in
inode_operations; ->i_link (let alone ->i_mode) isn't. As it is, we can easily tell symlinkinode_operationsfrom everything else on the source level. With your scheme we won't.
It could check a flag IOP_GET_LINK in ->i_opflags instead, so it would be the same number of checks. See patch below.
Benefits are that we get code that isn't actively misleading (via simple_get_link() existing but actually never being called), and filesystems can cache a symlink target in ->i_link if it becomes available later, i.e. if it's not immediately available at iget() time. Otherwise a filesystem-private field has to be used instead. (For fscrypt, I'd probably use fscrypt_info::ci_link.)
Anyway, if we're going to stick with the current approach we should at least add a comment to simple_get_link() explaining what it's really for...
fs/dcache.c +5 -6
@@ -1820,12 +1820,11 @@ static unsigned d_flags_for_inode(struct inode *inode) goto type_determined; } -if (unlikely (!(inode->i_opflags &IOP_NOFOLLOW ))) { -if (unlikely (inode->i_op->get_link)) { - add_flags =DCACHE_SYMLINK_TYPE ; - goto type_determined; - } - inode->i_opflags |=IOP_NOFOLLOW ; +if (unlikely (S_ISLNK (inode->i_mode))) { + add_flags =DCACHE_SYMLINK_TYPE ; +if (inode->i_op->get_link) + inode->i_opflags |=IOP_GET_LINK ; + goto type_determined; }if (unlikely (!S_ISREG (inode->i_mode)))
fs/ext4/symlink.c +0 -1
@@ -67,7 +67,6 @@ const struct inode_operations ext4_symlink_inode_operations = { };const struct inode_operations ext4_fast_symlink_inode_operations = { - .get_link = simple_get_link, .setattr = ext4_setattr, .getattr = ext4_getattr, .listxattr = ext4_listxattr,
fs/namei.c +2 -2
@@ -1067,7 +1067,7 @@ const char *get_link(struct nameidata *nd) nd->last_type =LAST_BIND ; res = inode->i_link; -if (!res) { +if (inode->i_opflags &IOP_GET_LINK ) {const char * (*get)(struct dentry *,struct inode *,struct delayed_call *); get = inode->i_op->get_link;
@@ -4730,7 +4730,7 @@ int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen) } link = inode->i_link; -if (!link) { +if (inode->i_opflags &IOP_GET_LINK ) { link = inode->i_op->get_link (dentry, inode, &done);if (IS_ERR (link))return PTR_ERR (link);
include/linux/fs.h +1 -1
@@ -604,7 +604,7 @@ is_uncached_acl(struct posix_acl *acl)#define IOP_FASTPERM 0x0001#define IOP_LOOKUP 0x0002 -#define IOP_NOFOLLOW 0x0004 +#define IOP_GET_LINK 0x0004#define IOP_XATTR 0x0008#define IOP_DEFAULT_READLINK 0x0010
- Eric
Al Viro
maintainerQuoted 2 lines · Eric Biggers
It could check a flag
IOP_GET_LINKin ->i_opflags instead, so it would be the same number of checks. See patch below.
With that patch ->i_link is completely unused if ->get_link() is non-NULL, so you get a method call on each traversal...
Quoted 5 lines
Benefits are that we get code that isn't actively misleading (via
simple_get_link()existing but actually never being called), and filesystems can cache a symlink target in ->i_link if it becomes available later, i.e. if it's not immediately available atiget()time. Otherwise a filesystem-private field has to be used instead. (For fscrypt, I'd probably usefscrypt_info::ci_link.)
What's to stop you from doing just that right now? You'd need to take care with barriers, but you'd need that anyway... As soon as ->i_link is set you'll get no more ->get_link() on that sucker, using the cached value from that point on. IDGI...
Eric Biggers
maintainerQuoted 8 lines · Al Viro
Quoted 2 lines · Eric Biggers
It could check a flag
IOP_GET_LINKin ->i_opflags instead, so it would be the same number of checks. See patch below.With that patch ->i_link is completely unused if ->get_link() is non-NULL, so you get a method call on each traversal...
.get_link would be left NULL in all inode_operations that currently use simple_get_link, then simple_get_link() would be removed. My example patch just changed it in ext4 as an example.
Quoted 10 lines
Quoted 5 lines
Benefits are that we get code that isn't actively misleading (via
simple_get_link()existing but actually never being called), and filesystems can cache a symlink target in ->i_link if it becomes available later, i.e. if it's not immediately available atiget()time. Otherwise a filesystem-private field has to be used instead. (For fscrypt, I'd probably usefscrypt_info::ci_link.)What's to stop you from doing just that right now? You'd need to take care with barriers, but you'd need that anyway... As soon as ->i_link is set you'll get no more ->get_link() on that sucker, using the cached value from that point on. IDGI...
1.) The VFS won't know to drop of RCU-walk mode, so waiting an RCU grace period
before freeing the symlink target becomes mandatory. (Which I'd like to do for fscrypt anyway, but doing it sanely appears to require implementing .destroy_inode() for ext4, f2fs, and ubifs. I hoped I could do non-RCU mode as a simpler first step.)
2.) The VFS won't know to use a read memory barrier when loading i_link.
The VFS could issue one unconditionally, but it would be unnecessary for regular fast symlinks.
- Eric
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Eric Biggers
maintainerQuoted 31 lines · Eric Biggers
Quoted 8 lines · Al Viro
Quoted 2 lines · Eric Biggers
It could check a flag
IOP_GET_LINKin ->i_opflags instead, so it would be the same number of checks. See patch below.With that patch ->i_link is completely unused if ->get_link() is non-NULL, so you get a method call on each traversal...
.get_link would be left NULL in all
inode_operationsthat currently usesimple_get_link, thensimple_get_link()would be removed. My example patch just changed it in ext4 as an example.Quoted 10 lines
Quoted 5 lines
Benefits are that we get code that isn't actively misleading (via
simple_get_link()existing but actually never being called), and filesystems can cache a symlink target in ->i_link if it becomes available later, i.e. if it's not immediately available atiget()time. Otherwise a filesystem-private field has to be used instead. (For fscrypt, I'd probably usefscrypt_info::ci_link.)What's to stop you from doing just that right now? You'd need to take care with barriers, but you'd need that anyway... As soon as ->i_link is set you'll get no more ->get_link() on that sucker, using the cached value from that point on. IDGI...
1.) The VFS won't know to drop of RCU-walk mode, so waiting an RCU grace period
before freeing the symlink target becomes mandatory. (Which I'd like to do for fscrypt anyway, but doing it sanely appears to require implementing .destroy_inode() for ext4, f2fs, and ubifs. I hoped I could do non-RCU mode as a simpler first step.)
Quoted 5 lines
2.) The VFS won't know to use a read memory barrier when loading
i_link.
The VFS could issue one unconditionally, but it would be unnecessary for regular fast symlinks.- Eric
Okay, actually all three filesystems have .destroy_inode() anyway. Not sure how I missed that. So it should be possible to free the decrypted symlink target from {ext4,f2fs,ubifs}_i_callback().
- Eric
Al Viro
maintainerQuoted 10 lines · Eric Biggers
Quoted 4 lines
What's to stop you from doing just that right now? You'd need to take care with barriers, but you'd need that anyway... As soon as ->i_link is set you'll get no more ->get_link() on that sucker, using the cached value from that point on. IDGI...
1.) The VFS won't know to drop of RCU-walk mode, so waiting an RCU grace period
before freeing the symlink target becomes mandatory. (Which I'd like to do for fscrypt anyway, but doing it sanely appears to require implementing .destroy_inode() for ext4, f2fs, and ubifs. I hoped I could do non-RCU mode as a simpler first step.)
You might want to check those filesystems. All three you've mentioned *have* ->destroy_inode() already.
Quoted 3 lines
2.) The VFS won't know to use a read memory barrier when loading
i_link.
The VFS could issue one unconditionally, but it would be unnecessary for regular fast symlinks.
Not really. All we need on the read side is READ_ONCE(); it will supply smp_read_barrier_depends() (which is a no-op except for alpha). On the write side we need smp_store_release() to set ->i_link (in addition to whatever serialization we want for actual calculation of the value to be cached, of course).
Eric Biggers
maintainerQuoted 7 lines · Al Viro
Quoted 5 lines · Eric Biggers
Quoted 4 lines
What's to stop you from doing just that right now? You'd need to take care with barriers, but you'd need that anyway... As soon as ->i_link is set you'll get no more ->get_link() on that sucker, using the cached value from that point on. IDGI...
Quoted 9 lines
Quoted 5 lines
1.) The VFS won't know to drop of RCU-walk mode, so waiting an RCU grace period
before freeing the symlink target becomes mandatory. (Which I'd like to do for fscrypt anyway, but doing it sanely appears to require implementing .destroy_inode() for ext4, f2fs, and ubifs. I hoped I could do non-RCU mode as a simpler first step.)You might want to check those filesystems. All three you've mentioned *have* ->destroy_inode() already.
Yep, I just noticed that.
Quoted 9 lines
Quoted 3 lines
2.) The VFS won't know to use a read memory barrier when loading
i_link.
The VFS could issue one unconditionally, but it would be unnecessary for regular fast symlinks.Not really. All we need on the read side is
READ_ONCE(); it will supplysmp_read_barrier_depends()(which is a no-op except for alpha). On the write side we needsmp_store_release()to set ->i_link (in addition to whatever serialization we want for actual calculation of the value to be cached, of course).
Okay, I didn't realize that READ_ONCE() would be sufficient. I thought smp_load_acquire() was needed. I guess you're right; we'd only read what the pointer points to, so it's a data dependency.
Do you see any problem with using cmpxchg_release() on the write side, so no additional lock is needed? (Like what we do for ->i_crypt_info, except currently it's actually cmpxchg() there, with a direct access on the read side. IIUC now, that should be changed to cmpxchg_release() and READ_ONCE().)
- Eric
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel