erofs-utils: mount: support mounting EROFS stored as an AWS S3 object
PATCH 1/2 erofs-utils: mount: support mounting EROFS stored as an AWS S3 object
Allow mount.erofs to directly mount an EROFS filesystem stored as an AWS S3 object without downloading it first: (meta)data is fetched on demand via HTTP range requests.
The source argument takes the form "bucket/key", e.g.:
$ mount.erofs -t erofs.nbd \
-o s3.endpoint=s3.amazonaws.com,s3.passwd_file=/path/to/passwd \
my-bucket/dir/foo.erofs mntIn addition, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables are also honored as fallback credentials.
--- lib/liberofs_s3.h | 5 + lib/remotes/s3.c | 303 ++++++++++++++++++++++++++++++++++++++++++++-- mkfs/main.c | 71 +---------- mount/main.c | 237 ++++++++++++++++++++++++++++++------ 4 files changed, 495 insertions(+), 121 deletions(-)
lib/liberofs_s3.h +5 -0
@@ -35,8 +35,13 @@ struct erofs_s3 {enum s3erofs_signature_version sig; }; +struct erofs_vfile ; +int s3erofs_build_trees (struct erofs_importer *im,struct erofs_s3 *s3,const char *path,bool fillzero); +struct erofs_vfile *s3erofs_io_open (struct erofs_s3 *s3,const char *bucket, +const char *key); +int s3erofs_parse_s3fs_passwd (const char *filepath,char *ak,char *sk); #ifdef __cplusplus }
lib/remotes/s3.c +291 -12
383 lines not on this page · raw message
mkfs/main.c +2 -69
83 lines not on this page · raw message
mount/main.c +197 -40
404 lines not on this page · raw message
Signature · 1 line
2.43.5
PATCH 2/2 erofs-utils: mount: add recovery support for S3 object mounts
--- lib/liberofs_s3.h | 7 ++- lib/remotes/s3.c | 72 ++++++++++++++++++++++++++++++ mount/main.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-)
lib/liberofs_s3.h +5 -2
@@ -26,8 +26,8 @@ enum s3erofs_signature_version {struct erofs_s3 {void *easy_curl; -const char *endpoint; -const char *region; +char *endpoint; +char *region;char access_key[S3_ACCESS_KEY_LEN +1 ];char secret_key[S3_SECRET_KEY_LEN +1 ];
@@ -43,6 +43,9 @@ struct erofs_vfile *s3erofs_io_open(struct erofs_s3 *s3, const char *bucket, const char *key);int s3erofs_parse_s3fs_passwd (const char *filepath,char *ak,char *sk); +char *s3erofs_encode_cred (const char *access_key,const char *secret_key); +int s3erofs_decode_cred (const char *b64,char **out_access_key,char **out_secret_key); + #ifdef __cplusplus }#endif
lib/remotes/s3.c +72 -0
78 lines not on this page · raw message
mount/main.c +109 -0
133 lines not on this page · raw message
Signature · 1 line
2.43.5
mount/main.c static int erofsmount_reattach(const char *target)
Quoted 930 lines · Gao Xiang
Allow mount.erofs to directly mount an EROFS filesystem stored as an AWS S3 object without downloading it first: (meta)data is fetched on demand via HTTP range requests.
The source argument takes the form "bucket/key", e.g.:
$ mount.erofs -t erofs.nbd \ -o s3.endpoint=s3.amazonaws.com,s3.passwd_file=/path/to/passwd \ my-bucket/dir/foo.erofs mntIn addition,
AWS_ACCESS_KEY_IDandAWS_SECRET_ACCESS_KEYenvironment variables are also honored as fallback credentials.Assisted-by qoder:(unknown)Cc Yuxuan Liu <[email protected]>Signed-off-by Gao Xiang <[email protected]>--- lib/liberofs_s3.h | 5 + lib/remotes/s3.c | 303 ++++++++++++++++++++++++++++++++++++++++++++-- mkfs/main.c | 71 +---------- mount/main.c | 237 ++++++++++++++++++++++++++++++------ 4 files changed, 495 insertions(+), 121 deletions(-)lib/liberofs_s3.h +5
-0@@ -35,8 +35,13 @@ struct erofs_s3 {enum s3erofs_signature_version sig; }; +struct erofs_vfile ; +int s3erofs_build_trees (struct erofs_importer *im,struct erofs_s3 *s3,const char *path,bool fillzero); +struct erofs_vfile *s3erofs_io_open (struct erofs_s3 *s3,const char *bucket, +const char *key); +int s3erofs_parse_s3fs_passwd (const char *filepath,char *ak,char *sk); #ifdef __cplusplus }lib/remotes/s3.c +291
-12@@ -20,6 +20,7 @@#include "erofs/importer.h" #include "liberofs_rebuild.h" #include "liberofs_s3.h" +#include "liberofs_base64.h" #define S3EROFS_PATH_MAX 1024#define S3EROFS_MAX_QUERY_PARAMS 16@@ -39,6 +40,7 @@ struct s3erofs_curl_request {char url[S3EROFS_URL_LEN ];char canonical_uri[S3EROFS_CANONICAL_URI_LEN ];char canonical_query[S3EROFS_CANONICAL_QUERY_LEN ]; +const char *method; };static const char *s3erofs_parse_host (const char *endpoint,const char **schema)@@ -353,6 +355,7 @@ static void s3erofs_to_hex(const u8 *data, size_t len, char *output)// See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTAuthentication.html#ConstructingTheAuthenticationHeader static char *s3erofs_sigv2_header(const struct curl_slist *headers, + const char *request_method, const char *content_md5, const char *content_type, const char *date, const char *canonical_uri, const char *ak,@@ -373,8 +376,8 @@ static char *s3erofs_sigv2_header(const struct curl_slist *headers,if (!canonical_uri) canonical_uri ="/" ; - pos =asprintf (&str,"GET\n%s\n%s\n%s\n%s%s" , content_md5, content_type, - date,"" , canonical_uri); + pos =asprintf (&str,"%s\n%s\n%s\n%s\n%s%s" , request_method, + content_md5, content_type, date,"" , canonical_uri);if (pos <0 )return ERR_PTR (-ENOMEM );@@ -401,6 +404,7 @@ free_string:// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html static char *s3erofs_sigv4_header (const struct curl_slist *headers, +const char *request_method,time_t request_time,const char *canonical_uri,const char *canonical_query,const char *region,const char *ak,const char *sk)@@ -430,13 +434,11 @@ static char *s3erofs_sigv4_header(const struct curl_slist *headers,// Task 1: Create canonical request if (asprintf (&canonical_request, -"GET\n" -"%s\n" -"%s\n" -"%s\n" +"%s\n%s\n%s\n%s\n" "host;x-amz-content-sha256;x-amz-date\n" "UNSIGNED-PAYLOAD" , - canonical_uri, canonical_query, canonical_headers) <0 ) { + request_method, canonical_uri, canonical_query, + canonical_headers) <0 ) { err = -ENOMEM ; goto err_canonical_headers; }@@ -533,7 +535,7 @@ static int s3erofs_request_insert_auth_v2(struct curl_slist **request_headers,s3erofs_format_time (time (NULL ), date +sizeof (date_prefix) -1 ,sizeof (date) -sizeof (date_prefix) +1 ,S3EROFS_DATE_RFC1123 ); - sigv2 =s3erofs_sigv2_header (*request_headers,NULL ,NULL , + sigv2 =s3erofs_sigv2_header (*request_headers, req->method,NULL ,NULL , date +sizeof (date_prefix) -1 , req->canonical_uri, s3->access_key, s3->secret_key);if (IS_ERR (sigv2))@@ -576,7 +578,7 @@ static int s3erofs_request_insert_auth_v4(struct curl_slist **request_headers, *request_headers =curl_slist_append (*request_headers, tmp);free (tmp); - sigv4 =s3erofs_sigv4_header (*request_headers, request_time, + sigv4 =s3erofs_sigv4_header (*request_headers, req->method, request_time, req->canonical_uri, req->canonical_query, s3->region, s3->access_key, s3->secret_key);if (IS_ERR (sigv4))@@ -619,6 +621,13 @@ static int s3erofs_request_perform(struct erofs_s3 *s3,long http_code =0 ;int ret; +if (!strcmp (req->method,"HEAD" )) { +curl_easy_setopt (curl,CURLOPT_NOBODY ,1L ); + }else { +curl_easy_setopt (curl,CURLOPT_NOBODY ,0L ); +curl_easy_setopt (curl,CURLOPT_HTTPGET ,1L ); + } +if (s3->access_key[0 ]) {if (s3->sig ==S3EROFS_SIGNATURE_VERSION_4 ) ret =s3erofs_request_insert_auth_v4 (&request_headers, req, s3);@@ -846,7 +855,7 @@ out:static int s3erofs_list_objects (struct s3erofs_object_iterator *it) { -struct s3erofs_curl_request req = {}; +struct s3erofs_curl_request req = { .method ="GET" , };struct s3erofs_curl_response resp = {};struct s3erofs_query_params params;struct erofs_s3 *s3 = it->s3;@@ -1014,7 +1023,7 @@ static int s3erofs_remote_getobject(struct erofs_importer *im, const char *bucket, const char *key) { struct erofs_sb_info *sbi = inode->sbi; -struct s3erofs_curl_request req = {}; +struct s3erofs_curl_request req = { .method ="GET" , };struct s3erofs_curl_getobject_resp resp;struct erofs_vfile vf;u64 diskbuf_off;@@ -1170,6 +1179,276 @@ err_global:return ret; } +struct s3erofs_vfile { +struct erofs_vfile vf; +struct erofs_s3 *s3; +char *bucket, *key; +u64 offset, size; +}; + +struct s3erofs_range_resp { +void *buf; +size_t len; +}; + +static size_t s3erofs_range_write_cb (void *contents,size_t size, +size_t nmemb,void *userp) +{ +struct s3erofs_range_resp *resp = userp; +size_t realsize = size * nmemb; + +if (realsize > resp->len) +return 0 ; + +memcpy (resp->buf, contents, realsize); + resp->buf = (char *)resp->buf + realsize; + resp->len -= realsize; +return realsize; +} + +static int s3erofs_get_object_range (struct s3erofs_vfile *s3vf, +void *buf,size_t len,u64 offset) +{ +struct s3erofs_curl_request req = { .method ="GET" , }; +struct erofs_s3 *s3 = s3vf->s3; +struct s3erofs_range_resp resp; +CURL *curl = s3->easy_curl; +u64 end = offset + len; +long http_code =0 ; +char range[64 ]; +int ret; + +if (end > s3vf->size) + end = s3vf->size; +if (__erofs_unlikely (end <= offset)) +return 0 ; + resp.buf = buf; + resp.len = end - offset; + + ret =s3erofs_prepare_url (&req, s3->endpoint, s3vf->bucket, + s3vf->key,NULL , s3->url_style, s3->sig); +if (ret <0 ) +return ret; + +/* Add Range header for partial content */ +snprintf (range,sizeof (range),"%llu-%llu" , offset |0ULL , (end -1 ) |0ULL ); + +curl_easy_setopt (curl,CURLOPT_RANGE , range); +curl_easy_setopt (curl,CURLOPT_WRITEFUNCTION , s3erofs_range_write_cb); + + ret =s3erofs_request_perform (s3, &req, &resp); +if (ret) +return ret; + + ret =curl_easy_getinfo (curl,CURLINFO_RESPONSE_CODE , &http_code); +if (ret !=CURLE_OK ) { +erofs_err ("curl_easy_getinfo() failed: %s" , +curl_easy_strerror (ret)); +return -EIO ; + } + +if (http_code !=206 && http_code !=200 ) { +erofs_err ("S3 range request failed with HTTP code %ld" , http_code); +return -EIO ; + } +return len - resp.len;/* actual bytes read */ +} + +static ssize_t s3erofs_io_pread (struct erofs_vfile *vf,void *buf, +size_t len,u64 offset) +{ +struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf; +int ret; + +if (offset >= s3vf->size) { +memset (buf,0 , len); +return len; + } + ret =s3erofs_get_object_range (s3vf, buf, len, offset); +if (ret >=0 && ret < len) { +memset (buf + ret,0 , len - ret); +return len; + } +return ret; +} + +static ssize_t s3erofs_io_read (struct erofs_vfile *vf,void *buf,size_t len) +{ +struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf; +ssize_t ret; + + ret =s3erofs_io_pread (vf, buf, len, s3vf->offset); +if (ret >0 ) + s3vf->offset += ret; +return ret; +} + +static void s3erofs_io_close (struct erofs_vfile *vf) +{ +struct s3erofs_vfile *s3vf = (struct s3erofs_vfile *)vf; + +if (!s3vf) +return ; + +s3erofs_curl_easy_exit (s3vf->s3); +free (s3vf->bucket); +free (s3vf->key); +free (s3vf); +} + +static struct erofs_vfops s3erofs_io_vfops = { + .pread = s3erofs_io_pread, + .read = s3erofs_io_read, + .close = s3erofs_io_close, +}; + +static int s3erofs_get_object_size (struct s3erofs_vfile *s3vf) +{ +struct s3erofs_curl_request req = { .method ="HEAD" , }; +struct erofs_s3 *s3 = s3vf->s3; +CURL *curl = s3->easy_curl; +long http_code =0 ; +double content_length =0 ; +int ret; + + ret =s3erofs_prepare_url (&req, s3->endpoint, s3vf->bucket, + s3vf->key,NULL , s3->url_style, s3->sig); +if (ret <0 ) +return ret; + + ret =s3erofs_request_perform (s3, &req,NULL ); +if (ret) +return ret; + + ret =curl_easy_getinfo (curl,CURLINFO_RESPONSE_CODE , &http_code); +if (ret !=CURLE_OK ) { +erofs_err ("curl_easy_getinfo() failed: %s" , +curl_easy_strerror (ret)); +return -EIO ; + } + +if (http_code !=200 ) { +erofs_err ("HEAD request failed with HTTP code %ld" , http_code); +return -EIO ; + } + + ret =curl_easy_getinfo (curl,CURLINFO_CONTENT_LENGTH_DOWNLOAD , + &content_length); +if (ret !=CURLE_OK ) +return -EIO ; + s3vf->size = (u64 )content_length; +return 0 ; +} + +struct erofs_vfile *s3erofs_io_open (struct erofs_s3 *s3,const char *bucket, +const char *key) +{ +struct s3erofs_vfile *s3vf; +int ret = -ENOMEM ; + + s3vf =calloc (1 ,sizeof (*s3vf)); +if (!s3vf) +return ERR_PTR (-ENOMEM ); + + s3vf->vf = (struct erofs_vfile ){.ops = &s3erofs_io_vfops}; + s3vf->bucket =strdup (bucket); +if (!s3vf->bucket) + goto err_free; + s3vf->key =strdup (key); +if (!s3vf->key) + goto err_free; + s3vf->s3 = s3; + + ret =s3erofs_curl_easy_init (s3vf->s3); +if (ret) + goto err_free; + +/* Get object size via HEAD request */ + ret =s3erofs_get_object_size (s3vf); +if (ret) { +erofs_err ("failed to get S3 object size" ); + goto err_curl; + } + +erofs_dbg ("S3 object (%s) size: %llu" , s3vf->key, s3vf->size); +return &s3vf->vf; + +err_curl: +s3erofs_curl_easy_exit (s3); +err_free: +free (s3vf->key); +free (s3vf->bucket); +free (s3vf); +return ERR_PTR (ret); +} + +int s3erofs_parse_s3fs_passwd (const char *filepath,char *ak,char *sk) +{ +char buf[S3_ACCESS_KEY_LEN +S3_SECRET_KEY_LEN +3 ]; +struct stat st; +int fd, n, ret; +char *colon; + + fd =open (filepath,O_RDONLY ); +if (fd <0 ) { +erofs_err ("failed to open passwd_file %s" , filepath); +return -errno; + } + + ret =fstat (fd, &st); +if (ret) { + ret = -errno; + goto err; + } + +if (!S_ISREG (st.st_mode)) { +erofs_err ("%s is not a regular file" , filepath); + ret = -EINVAL ; + goto err; + } + +if ((st.st_mode &077 ) !=0 ) +erofs_warn ("passwd_file %s should not be accessible by group or others" , + filepath); + +if (st.st_size >=sizeof (buf)) { +erofs_err ("passwd_file %s is too large (size: %llu)" , filepath, + st.st_size |0ULL ); + ret = -EINVAL ; + goto err; + } + + n =read (fd, buf, st.st_size); +if (n <0 ) { + ret = -errno; + goto err; + } + buf[n] ='\0' ; + +while (n >0 && (buf[n -1 ] =='\n' || buf[n -1 ] =='\r' )) + buf[--n] ='\0' ; + + colon =strchr (buf,':' ); +if (!colon) { + ret = -EINVAL ; + goto err; + } + *colon ='\0' ; + +if (strlen (buf) >S3_ACCESS_KEY_LEN || +strlen (colon +1 ) >S3_SECRET_KEY_LEN ) { + ret = -EINVAL ; + goto err; + } + +strcpy (ak, buf); +strcpy (sk, colon +1 ); + +err: +close (fd); +return ret; +} + #ifdef TEST struct s3erofs_prepare_url_testcase { const char *name;@@ -1186,7 +1465,7 @@ struct s3erofs_prepare_url_testcase {static bool run_s3erofs_prepare_url_test (const struct s3erofs_prepare_url_testcase *tc,enum s3erofs_signature_version sig) { -struct s3erofs_curl_request req = {}; +struct s3erofs_curl_request req = { .method ="GET" , };int ret;const char *expected_canonical;mkfs/main.c +2
-69@@ -634,73 +634,6 @@ static void mkfs_parse_tar_cfg(char *cfg) }#ifdef S3EROFS_ENABLED -static int mkfs_parse_s3_cfg_passwd (const char *filepath,char *ak,char *sk) -{ -struct stat st; -int fd, n, ret; -char buf[S3_ACCESS_KEY_LEN +S3_SECRET_KEY_LEN +3 ]; -char *colon; - - fd =open (filepath,O_RDONLY ); -if (fd <0 ) { -erofs_err ("failed to open passwd_file %s" , filepath); -return -errno; - } - - ret =fstat (fd, &st); -if (ret) { - ret = -errno; - goto err; - } - -if (!S_ISREG (st.st_mode)) { -erofs_err ("%s is not a regular file" , filepath); - ret = -EINVAL ; - goto err; - } - -if ((st.st_mode &077 ) !=0 ) -erofs_warn ("passwd_file %s should not be accessible by group or others" , - filepath); - -if (st.st_size >=sizeof (buf)) { -erofs_err ("passwd_file %s is too large (size: %llu)" , filepath, - st.st_size |0ULL ); - ret = -EINVAL ; - goto err; - } - - n =read (fd, buf, st.st_size); -if (n <0 ) { - ret = -errno; - goto err; - } - buf[n] ='\0' ; - -while (n >0 && (buf[n -1 ] =='\n' || buf[n -1 ] =='\r' )) - buf[--n] ='\0' ; - - colon =strchr (buf,':' ); -if (!colon) { - ret = -EINVAL ; - goto err; - } - *colon ='\0' ; - -if (strlen (buf) >S3_ACCESS_KEY_LEN || -strlen (colon +1 ) >S3_SECRET_KEY_LEN ) { - ret = -EINVAL ; - goto err; - } - -strcpy (ak, buf); -strcpy (sk, colon +1 ); - -err: -close (fd); -return ret; -} -static int mkfs_parse_s3_cfg (char *cfg_str) {char *p, *q, *opt;@@ -734,8 +667,8 @@ static int mkfs_parse_s3_cfg(char *cfg_str)if ((p =strstr (opt,"passwd_file=" ))) { p +=sizeof ("passwd_file=" ) -1 ; - ret =mkfs_parse_s3_cfg_passwd (p, s3cfg.access_key, - s3cfg.secret_key); + ret =s3erofs_parse_s3fs_passwd (p, s3cfg.access_key, + s3cfg.secret_key);if (ret)return ret; }else if ((p =strstr (opt,"urlstyle=" ))) {mount/main.c +194
-37@@ -22,6 +22,7 @@#ifdef EROFS_FANOTIFY_ENABLED #include "../lib/liberofs_fanotify.h" #endif +#include "../lib/liberofs_s3.h" #ifdef HAVE_LINUX_LOOP_H #include <linux/loop.h> @@ -88,13 +89,17 @@ static struct erofsmount_cfg {enum erofsmount_source_type {EROFSMOUNT_SOURCE_LOCAL ,EROFSMOUNT_SOURCE_OCI , +EROFSMOUNT_SOURCE_S3_OBJECT , };static struct erofsmount_source {enum erofsmount_source_type type;union { -const char *device_path;struct ocierofs_config ocicfg; +struct { +const char *device_path; +struct erofs_s3 s3cfg; + }; }; } mountsrc;@@ -104,27 +109,36 @@ static void usage(int argc, char **argv) "Manage EROFS filesystem.\n" "\n" "General options:\n" -" -V, --version print the version number of mount.erofs and exit\n" -" -h, --help display this help and exit\n" -" -d <0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n" -" -o options comma-separated list of mount options\n" -" -t type[.subtype] filesystem type (and optional subtype)\n" -" subtypes: fuse, local, nbd" EROFSMOUNT_FANOTIFY_HELP "\n" -" -u unmount the filesystem\n" -" --disconnect abort an existing NBD device forcibly\n" -" --reattach reattach to an existing NBD device\n" + " -V, --version print the version number of mount.erofs and exit\n" + " -h, --help display this help and exit\n" + " -d <0-9> set output verbosity; 0=quiet, 9=verbose (default=%i)\n" + " -o options comma-separated list of mount options\n" + " -t type[.subtype] filesystem type (and optional subtype)\n" + " subtypes: fuse, local, nbd" EROFSMOUNT_FANOTIFY_HELP "\n" + " -u unmount the filesystem\n" + " --disconnect abort an existing NBD device forcibly\n" + " --reattach reattach to an existing NBD device\n" #ifdef OCIEROFS_ENABLED "\n" "OCI-specific options (EXPERIMENTAL, with -o):\n" -" oci.blob=<digest> specify OCI blob digest (sha256:...)\n" -" oci.layer=<index> specify OCI layer index\n" -" oci.platform=<name> specify platform (default: linux/amd64)\n" -" oci.username=<user> username for authentication (optional)\n" -" oci.password=<pass> password for authentication (optional)\n" -" oci.tarindex=<path> path to tarball index file (optional)\n" -" oci.zinfo=<path> path to gzip zinfo file (optional)\n" -" oci.insecure use HTTP instead of HTTPS (optional)\n" + " oci.blob=<digest> specify OCI blob digest (sha256:...)\n" + " oci.layer=<index> specify OCI layer index\n" + " oci.platform=<name> specify platform (default: linux/amd64)\n" + " oci.username=<user> username for authentication (optional)\n" + " oci.password=<pass> password for authentication (optional)\n" + " oci.tarindex=<path> path to tarball index file (optional)\n" + " oci.zinfo=<path> path to gzip zinfo file (optional)\n" + " oci.insecure use HTTP instead of HTTPS (optional)\n" #endif +#ifdef S3EROFS_ENABLED +"\n" +"S3-specific options (EXPERIMENTAL, with -o):\n" +" s3.endpoint=<url> S3 endpoint URL (e.g., s3.amazonaws.com)\n" +" s3.passwd_file=<path> specify a s3fs-compatible password file\n" +" s3.region=<region> region code in which endpoint belongs to (required for sigv4)\n" +" s3.sig=<2,4> S3 API signature version (default: 2)\n" +" s3.urlstyle=<vhost|path> S3 API calling URL (default: vhost)\n" + #endif , argv[0], EROFS_WARN); }@@ -210,6 +224,85 @@ static int erofsmount_parse_oci_option(const char *option) }#endif +#ifdef S3EROFS_ENABLED +static int erofsmount_parse_s3_option (const char *option,struct erofs_s3 *s3cfg) +{ +const char *p; +int ret; + +if ((p =strstr (option,"s3.endpoint=" )) !=NULL ) { + p +=sizeof ("s3.endpoint=" ) -1 ; + s3cfg->endpoint =strdup (p); +if (!s3cfg->endpoint) +return -ENOMEM ; + }else if ((p =strstr (option,"s3.passwd_file=" )) !=NULL ) { + p +=sizeof ("s3.passwd_file=" ) -1 ; + ret =s3erofs_parse_s3fs_passwd (p, s3cfg->access_key, + s3cfg->secret_key); +if (ret) +return ret; + }else if ((p =strstr (option,"s3.region=" )) !=NULL ) { + p +=sizeof ("s3.region=" ) -1 ; + s3cfg->region =strdup (p); +if (!s3cfg->region) +return -ENOMEM ; + }else if ((p =strstr (option,"s3.urlstyle=" )) !=NULL ) { + p +=sizeof ("s3.urlstyle=" ) -1 ; +if (!strcmp (p,"vhost" )) + s3cfg->url_style =S3EROFS_URL_STYLE_VIRTUAL_HOST ; +else if (!strcmp (p,"path" )) + s3cfg->url_style =S3EROFS_URL_STYLE_PATH ; +else { +erofs_err ("invalid S3 URL style %s" , p); +return -EINVAL ; + } + }else if ((p =strstr (option,"s3.sig=" )) !=NULL ) { + p +=sizeof ("s3.sig=" ) -1 ; +if (!strcmp (p,"2" )) + s3cfg->sig =S3EROFS_SIGNATURE_VERSION_2 ; +else if (!strcmp (p,"4" )) + s3cfg->sig =S3EROFS_SIGNATURE_VERSION_4 ; +else { +erofs_err ("invalid S3 signature version %s" , p); +return -EINVAL ; + } + }else { +return -EINVAL ; + } +return 0 ; +} + +static int erofsmount_parse_s3_source (struct erofs_s3 *s3cfg,const char *source, +char **bucket,char **key) +{ +const char *slash; + +if (!source || !*source) +return -EINVAL ; + + slash =strchr (source,'/' ); +if (!slash) { +/* No slash: treat entire source as bucket, empty key */ + *bucket =strdup (source); + *key =strdup ("" ); + }else { + *bucket =strndup (source, slash - source); + *key =strdup (slash +1 ); + } +if (!*bucket || !*key) { +free (*bucket); +free (*key); +return -ENOMEM ; + } +return 0 ; +} +#else +static int erofsmount_parse_s3_option (const char *option,void *s3cfg) +{ +return -EINVAL ; +} +#endif +static long erofsmount_parse_flagopts (char *s,long flags,char **more) { static const struct {@@ -253,6 +346,33 @@ static long erofsmount_parse_flagopts(char *s, long flags, char **more) err =erofsmount_parse_oci_option (s);if (err <0 )return err; +#ifdef S3EROFS_ENABLED + }else if (strncmp (s,"s3." ,3 ) ==0 ) { +/* Initialize s3cfg here iff != EROFSMOUNT_SOURCE_S3_OBJECT */ +if (mountsrc.type !=EROFSMOUNT_SOURCE_S3_OBJECT ) { +erofs_warn ("EXPERIMENTAL S3 mount support in use, use at your own risk." ); + mountsrc.type =EROFSMOUNT_SOURCE_S3_OBJECT ; + mountsrc.s3cfg.url_style =S3EROFS_URL_STYLE_VIRTUAL_HOST ; + mountsrc.s3cfg.sig =S3EROFS_SIGNATURE_VERSION_2 ; + mountsrc.s3cfg.access_key[0 ] ='\0' ; + mountsrc.s3cfg.secret_key[0 ] ='\0' ; +if (getenv ("AWS_ACCESS_KEY_ID" )) { +strncpy (mountsrc.s3cfg.access_key, +getenv ("AWS_ACCESS_KEY_ID" ), +S3_ACCESS_KEY_LEN ); + mountsrc.s3cfg.access_key[S3_ACCESS_KEY_LEN ] ='\0' ; + } +if (getenv ("AWS_SECRET_ACCESS_KEY" )) { +strncpy (mountsrc.s3cfg.secret_key, +getenv ("AWS_SECRET_ACCESS_KEY" ), +S3_SECRET_KEY_LEN ); + mountsrc.s3cfg.secret_key[S3_SECRET_KEY_LEN ] ='\0' ; + } + } + err =erofsmount_parse_s3_option (s, &mountsrc.s3cfg); +if (err <0 ) +return err; +#endif }else {for (i =0 ; i <ARRAY_SIZE (opts); ++i) {if (!strcasecmp (s, opts[i].name)) {@@ -635,8 +755,9 @@ err_out: }struct erofsmount_nbd_ctx { -struct erofs_vfile vd;/* virtual device */ +struct erofs_vfile _vd;/* virtual device */ struct erofs_vfile sk;/* socket file */ +struct erofs_vfile *vd; };static void *erofsmount_nbd_loopfn (void *arg)@@ -666,7 +787,7 @@ static void *erofsmount_nbd_loopfn(void *arg)erofs_nbd_send_reply_header (ctx->sk.fd, rq.cookie,0 ); pos = rq.from; do { - written = erofs_io_sendfile(&ctx->sk, &ctx->vd, &pos, rq.len); + written = erofs_io_sendfile(&ctx->sk, ctx->vd, &pos, rq.len); if (written == -EINTR) { err = written; goto out;@@ -680,49 +801,68 @@ static void *erofsmount_nbd_loopfn(void *arg) } } out: -erofs_io_close (&ctx->vd); +erofs_io_close (ctx->vd);erofs_io_close (&ctx->sk);return (void *)(uintptr_t )err; }static int erofsmount_startnbd (int nbdfd,struct erofsmount_source *source) { -struct erofsmount_nbd_ctx ctx = {}; +struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};uintptr_t retcode;pthread_t th;int err, err2;if (source->type ==EROFSMOUNT_SOURCE_OCI ) {if (source->ocicfg.tarindex_path || source->ocicfg.zinfo_path) { - err =erofsmount_tarindex_open (&ctx.vd, &source->ocicfg, + err =erofsmount_tarindex_open (ctx.vd, &source->ocicfg, source->ocicfg.tarindex_path, source->ocicfg.zinfo_path);if (err) goto out_closefd; }else { - err =ocierofs_io_open (&ctx.vd, &source->ocicfg); + err =ocierofs_io_open (ctx.vd, &source->ocicfg);if (err) goto out_closefd; } +#ifdef S3EROFS_ENABLED + }else if (source->type ==EROFSMOUNT_SOURCE_S3_OBJECT ) { +char *bucket =NULL , *key =NULL ; +struct erofs_vfile *s3vf; + + err =erofsmount_parse_s3_source (&source->s3cfg, source->device_path, + &bucket, &key); +if (err) + goto out_closefd; + + s3vf =s3erofs_io_open (&source->s3cfg, bucket, key); +free (bucket); +free (key); +if (IS_ERR (s3vf)) { + err =PTR_ERR (s3vf); + goto out_closefd; + } + ctx.vd = s3vf; +#endif }else { err =open (source->device_path,O_RDONLY );if (err <0 ) { err = -errno; goto out_closefd; } - ctx.vd.fd = err; + ctx._vd.fd = err; } err =erofs_nbd_connect (nbdfd,9 ,EROFSMOUNT_NBD_DISK_SIZE );if (err <0 ) { -erofs_io_close (&ctx.vd); +erofs_io_close (ctx.vd); goto out_closefd; } ctx.sk.fd = err; err = -pthread_create (&th,NULL , erofsmount_nbd_loopfn, &ctx);if (err) { -erofs_io_close (&ctx.vd); +erofs_io_close (ctx.vd);erofs_io_close (&ctx.sk); goto out_closefd; }@@ -840,7 +980,7 @@ static char *erofsmount_write_recovery_info(struct erofsmount_source *source)if (source->type ==EROFSMOUNT_SOURCE_OCI ) err =erofsmount_write_recovery_oci (f, source); -else +else if (source->type ==EROFSMOUNT_SOURCE_LOCAL ) err =erofsmount_write_recovery_local (f, source);fclose (f);@@ -996,12 +1136,12 @@ static int erofsmount_reattach_gzran_oci(struct erofsmount_nbd_ctx *ctx,if (err <0 )return -ENOMEM ; - err =erofsmount_reattach_oci (&ctx->vd,"OCI_NATIVE_BLOB" , oci_source); + err =erofsmount_reattach_oci (ctx->vd,"OCI_NATIVE_BLOB" , oci_source);free (oci_source);if (err)return err; - temp_vd = ctx->vd; + temp_vd = *ctx->vd; oci_cfg.image_ref =strdup (source);if (!oci_cfg.image_ref) {erofs_io_close (&temp_vd);@@ -1013,7 +1153,7 @@ static int erofsmount_reattach_gzran_oci(struct erofsmount_nbd_ctx *ctx,if (token_count >4 && tokens[4 ] && *tokens[4 ]) zinfo_path = tokens[4 ]; - err =erofsmount_tarindex_open (&ctx->vd, &oci_cfg, + err =erofsmount_tarindex_open (ctx->vd, &oci_cfg, meta_path, zinfo_path);free (oci_cfg.image_ref);erofs_io_close (&temp_vd);@@ -1056,7 +1196,7 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source)return -errno;if ((*pid =fork ()) ==0 ) { -struct erofsmount_nbd_ctx ctx = {}; +struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};char *recp;/* Otherwise, NBD disconnect sends SIGPIPE, skipping cleanup */ @@ -1065,25 +1205,42 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source)if (source->type ==EROFSMOUNT_SOURCE_OCI ) {if (source->ocicfg.tarindex_path || source->ocicfg.zinfo_path) { - err =erofsmount_tarindex_open (&ctx.vd, &source->ocicfg, + err =erofsmount_tarindex_open (ctx.vd, &source->ocicfg, source->ocicfg.tarindex_path, source->ocicfg.zinfo_path);if (err)exit (EXIT_FAILURE ); }else { - err =ocierofs_io_open (&ctx.vd, &source->ocicfg); + err =ocierofs_io_open (ctx.vd, &source->ocicfg);if (err)exit (EXIT_FAILURE ); } +#ifdef S3EROFS_ENABLED + }else if (source->type ==EROFSMOUNT_SOURCE_S3_OBJECT ) { +char *bucket =NULL , *key =NULL ; +struct erofs_vfile *s3vf; + + err =erofsmount_parse_s3_source (&source->s3cfg, source->device_path, + &bucket, &key); +if (err) +exit (EXIT_FAILURE ); + + s3vf =s3erofs_io_open (&source->s3cfg, bucket, key); +free (bucket); +free (key); +if (IS_ERR (s3vf)) +exit (EXIT_FAILURE ); + ctx.vd = s3vf; +#endif }else { err =open (source->device_path,O_RDONLY );if (err <0 )exit (EXIT_FAILURE ); - ctx.vd.fd = err; + ctx._vd.fd = err; } recp =erofsmount_write_recovery_info (source);if (IS_ERR (recp)) { -erofs_io_close (&ctx.vd); +erofs_io_close (ctx.vd);exit (EXIT_FAILURE ); }@@ -1106,7 +1263,7 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source) } } } -erofs_io_close (&ctx.vd); +erofs_io_close (ctx.vd); out_fork: (void )unlink (recp);free (recp);@@ -1186,13 +1343,13 @@ static int erofsmount_reattach(const char *target)
In erofsmount_reattach() we should:
struct erofsmount_nbd_ctx ctx = {}; => struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};
otherwise uninitialized ctx.vd leads to segfault.
Thanks,
Yifan Zhao
Quoted 24 lines
mount/main.c +3
-3err = -errno; goto err_line; } - ctx.vd.fd = err; + ctx.vd->fd = err; }else if (!strcmp (line,"TARINDEX_OCI_BLOB" )) { err =erofsmount_reattach_gzran_oci (&ctx, source);if (err) goto err_line; }else if (!strcmp (line,"OCI_LAYER" ) || !strcmp (line,"OCI_NATIVE_BLOB" )) { - err =erofsmount_reattach_oci (&ctx.vd, line, source); + err =erofsmount_reattach_oci (ctx.vd, line, source);if (err) goto err_line; }else {@@ -1214,7 +1371,7 @@ static int erofsmount_reattach(const char *target)erofs_io_close (&ctx.sk); err =0 ; } -erofs_io_close (&ctx.vd); +erofs_io_close (ctx.vd); err_line:free (line); err_identifier:
Quoted 2 lines · Yifan Zhao
On 4/17/2026 6:18 PM, Gao Xiang wrote:
...
Quoted 13 lines
(quoted diff) +0
-0@@ -1106,7 +1263,7 @@ static int erofsmount_startnbd_nl(pid_t *pid, struct erofsmount_source *source) > } > } > } > - erofs_io_close(&ctx.vd); > + erofs_io_close(ctx.vd);Review static int erofsmount_reattach(const char *target)
Quoted 4 lines
out_fork: (void)unlink(recp); free(recp);(quoted diff) +0
-0@@ -1186,13 +1343,13 @@ static int erofsmount_reattach(const char *target)In
erofsmount_reattach()we should:
Quoted 4 lines
struct erofsmount_nbd_ctx ctx = {};=>struct erofsmount_nbd_ctx ctx = {.vd = &ctx._vd};otherwise uninitialized ctx.vd leads to segfault.
Thanks, fixed.
Thanks,
Gao Xiang
Quoted 6 lines
Thanks,
Yifan Zhao