replace CURLOPT_FILE With CURLOPT_WRITEDATA
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 2/3 replace CURLOPT_FILE With CURLOPT_WRITEDATA
CURLOPT_FILE has been deprecated since 2003.
--- http-push.c | 6 +++--- http-walker.c | 2 +- http.c | 6 +++--- remote-curl.c | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-)
http-push.c +3 -3
@@ -894,7 +894,7 @@ static struct remote_lock *lock_remote(const char *path, long timeout) slot->results = &results;curl_setup_http (slot->curl, url,DAV_LOCK , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer); lock =xcalloc (1 ,sizeof (*lock)); lock->timeout =-1 ;
@@ -1151,7 +1151,7 @@ static void remote_ls(const char *path, int flags,curl_setup_http (slot->curl, url,DAV_PROPFIND , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer);if (start_active_slot (slot)) {run_active_slot (slot);
@@ -1225,7 +1225,7 @@ static int locking_available(void)curl_setup_http (slot->curl, repo->url,DAV_PROPFIND , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer);if (start_active_slot (slot)) {run_active_slot (slot);
http-walker.c +1 -1
@@ -384,7 +384,7 @@ static void fetch_alternates(struct walker *walker, const char *base) alt_req.walker = walker; slot->callback_data = &alt_req; -curl_easy_setopt (slot->curl,CURLOPT_FILE , &buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &buffer);curl_easy_setopt (slot->curl,CURLOPT_WRITEFUNCTION , fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_URL , url.buf);
http.c +3 -3
@@ -1921,7 +1921,7 @@ static int http_request(const char *url,curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,1 ); }else {curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,0 ); -curl_easy_setopt (slot->curl,CURLOPT_FILE , result); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , result);if (target ==HTTP_REQUEST_FILE ) {off_t posn =ftello (result);
@@ -2337,7 +2337,7 @@ struct http_pack_request *new_direct_http_pack_request( } preq->slot =get_active_slot (); -curl_easy_setopt (preq->slot->curl,CURLOPT_FILE , preq->packfile); +curl_easy_setopt (preq->slot->curl,CURLOPT_WRITEDATA , preq->packfile);curl_easy_setopt (preq->slot->curl,CURLOPT_WRITEFUNCTION , fwrite);curl_easy_setopt (preq->slot->curl,CURLOPT_URL , preq->url); curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
@@ -2508,7 +2508,7 @@ struct http_object_request *new_http_object_request(const char *base_url, freq->slot =get_active_slot (); -curl_easy_setopt (freq->slot->curl,CURLOPT_FILE , freq); +curl_easy_setopt (freq->slot->curl,CURLOPT_WRITEDATA , freq);curl_easy_setopt (freq->slot->curl,CURLOPT_FAILONERROR ,0 );curl_easy_setopt (freq->slot->curl,CURLOPT_WRITEFUNCTION , fwrite_sha1_file);curl_easy_setopt (freq->slot->curl,CURLOPT_ERRORBUFFER , freq->errorstr);
remote-curl.c +2 -2
@@ -847,7 +847,7 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results)curl_easy_setopt (slot->curl,CURLOPT_POSTFIELDSIZE ,4 );curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , headers);curl_easy_setopt (slot->curl,CURLOPT_WRITEFUNCTION , fwrite_buffer); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &buf); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &buf); err =run_slot (slot, results);
@@ -1012,7 +1012,7 @@ static int post_rpc(struct rpc_state *rpc, int stateless_connect, int flush_rece rpc_in_data.slot = slot; rpc_in_data.check_pktline = stateless_connect;memset (&rpc_in_data.pktline_state,0 ,sizeof (rpc_in_data.pktline_state)); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &rpc_in_data); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &rpc_in_data);curl_easy_setopt (slot->curl,CURLOPT_FAILONERROR ,0 );
Signature · 1 line
2.28.0.1011.ga647a8990f-goog
Hi Sean,
Quoted 1 line · Sean McAllister
CURLOPT_FILEhas been deprecated since 2003.
To be precise, it has been aliased to CURLOPT_WRITEDATA in cURL v7.9.7:
https://github.com/curl/curl/commit/980a47b42b95d7b9ff3378dc7b0f2e1c453fb649
The CURLOPT_WRITEDATA symbol became the preferred one only in v7.37.1 in 2014, though:
https://github.com/curl/curl/commit/5fcef972b289bdc7f3dbd7a55a5ada0460b74b2d
Ciao,
Dscho
Quoted 110 lines
Signed-off-by Sean McAllister <[email protected]>--- http-push.c | 6 +++--- http-walker.c | 2 +- http.c | 6 +++--- remote-curl.c | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-)http-push.c +3
-3@@ -894,7 +894,7 @@ static struct remote_lock *lock_remote(const char *path, long timeout) slot->results = &results;curl_setup_http (slot->curl, url,DAV_LOCK , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer); lock =xcalloc (1 ,sizeof (*lock)); lock->timeout =-1 ;@@ -1151,7 +1151,7 @@ static void remote_ls(const char *path, int flags,curl_setup_http (slot->curl, url,DAV_PROPFIND , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer);if (start_active_slot (slot)) {run_active_slot (slot);@@ -1225,7 +1225,7 @@ static int locking_available(void)curl_setup_http (slot->curl, repo->url,DAV_PROPFIND , &out_buffer, fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , dav_headers); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &in_buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &in_buffer);if (start_active_slot (slot)) {run_active_slot (slot);http-walker.c +1
-1@@ -384,7 +384,7 @@ static void fetch_alternates(struct walker *walker, const char *base) alt_req.walker = walker; slot->callback_data = &alt_req; -curl_easy_setopt (slot->curl,CURLOPT_FILE , &buffer); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &buffer);curl_easy_setopt (slot->curl,CURLOPT_WRITEFUNCTION , fwrite_buffer);curl_easy_setopt (slot->curl,CURLOPT_URL , url.buf);http.c +3
-3@@ -1921,7 +1921,7 @@ static int http_request(const char *url,curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,1 ); }else {curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,0 ); -curl_easy_setopt (slot->curl,CURLOPT_FILE , result); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , result);if (target ==HTTP_REQUEST_FILE ) {off_t posn =ftello (result);@@ -2337,7 +2337,7 @@ struct http_pack_request *new_direct_http_pack_request( } preq->slot =get_active_slot (); -curl_easy_setopt (preq->slot->curl,CURLOPT_FILE , preq->packfile); +curl_easy_setopt (preq->slot->curl,CURLOPT_WRITEDATA , preq->packfile);curl_easy_setopt (preq->slot->curl,CURLOPT_WRITEFUNCTION , fwrite);curl_easy_setopt (preq->slot->curl,CURLOPT_URL , preq->url); curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,@@ -2508,7 +2508,7 @@ struct http_object_request *new_http_object_request(const char *base_url, freq->slot =get_active_slot (); -curl_easy_setopt (freq->slot->curl,CURLOPT_FILE , freq); +curl_easy_setopt (freq->slot->curl,CURLOPT_WRITEDATA , freq);curl_easy_setopt (freq->slot->curl,CURLOPT_FAILONERROR ,0 );curl_easy_setopt (freq->slot->curl,CURLOPT_WRITEFUNCTION , fwrite_sha1_file);curl_easy_setopt (freq->slot->curl,CURLOPT_ERRORBUFFER , freq->errorstr);remote-curl.c +2
-3@@ -847,7 +847,7 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results)curl_easy_setopt (slot->curl,CURLOPT_POSTFIELDSIZE ,4 );curl_easy_setopt (slot->curl,CURLOPT_HTTPHEADER , headers);curl_easy_setopt (slot->curl,CURLOPT_WRITEFUNCTION , fwrite_buffer); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &buf); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &buf); err =run_slot (slot, results);@@ -1012,7 +1012,7 @@ static int post_rpc(struct rpc_state *rpc, int stateless_connect, int flush_rece rpc_in_data.slot = slot; rpc_in_data.check_pktline = stateless_connect;memset (&rpc_in_data.pktline_state,0 ,sizeof (rpc_in_data.pktline_state)); -curl_easy_setopt (slot->curl,CURLOPT_FILE , &rpc_in_data); +curl_easy_setopt (slot->curl,CURLOPT_WRITEDATA , &rpc_in_data);curl_easy_setopt (slot->curl,CURLOPT_FAILONERROR ,0 ); --2.28.0.1011.ga647a8990f-goog
PATCH 3/3 http: automatically retry some requests
In reply to a message not in the archiveSome HTTP response codes indicate a server state that can support retrying the request rather than immediately erroring out. The server can also provide information about how long to wait before retries to via the Retry-After header. So check the server response and retry some reasonable number of times before erroring out to better accomodate transient errors.
Exiting immediately becomes irksome when pulling large multi-repo code bases such as Android or Chromium, as often the entire fetch operation has to be restarted from the beginning due to an error in one repo. If we can reduce how often that occurs, then it's a big win.
--- Documentation/config/http.txt | 5 ++ http.c | 121 +++++++++++++++++++++++++++++++++- http.h | 13 +++- remote-curl.c | 2 +- t/t5539-fetch-http-shallow.sh | 41 ++++++++---- t/t5540-http-push-webdav.sh | 26 +++++++- t/t5541-http-push-smart.sh | 15 +++++ t/t5550-http-fetch-dumb.sh | 12 ++++ t/t5551-http-fetch-smart.sh | 14 ++++ t/t5601-clone.sh | 6 +- 10 files changed, 237 insertions(+), 18 deletions(-)
Documentation/config/http.txt +5 -0
@@ -260,6 +260,11 @@ http.followRedirects:: the base for the follow-up requests, this is generally sufficient. The default is `initial`. +http.retryLimit:: + Some HTTP error codes (eg: 429,503) can reasonably be retried if + they're encountered. This value configures the number of retry attempts + before giving up. The default retry limit is 3. + http.<url>.*:: Any of the http.* options above can be applied selectively to some URLs. For a config key to match a URL, each element of the config key is
http.c +118 -3
182 lines not on this page · raw message
http.h +12 -1
@@ -86,6 +86,17 @@ size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf);curlioerr ioctl_buffer (CURL *handle,int cmd,void *clientp);#endif +/* +* Query the value of an HTTP header. +* +* If the header is found, then a newly allocate string is returned through +* the value parameter, and the length is returned. +* +* If not found, returns 0 +*/ +size_t http_header_value ( +const struct strbuf headers,const char *header,char **value); +/* Slot lifecycle functions */ struct active_request_slot *get_active_slot (void );int start_active_slot (struct active_request_slot *slot);
@@ -99,7 +110,7 @@ void finish_all_active_slots(void);* */ int run_one_slot (struct active_request_slot *slot, -struct slot_results *results); +struct slot_results *results,int *http_code);#ifdef USE_CURL_MULTI void fill_active_slots (void );
remote-curl.c +1 -1
@@ -805,7 +805,7 @@ static int run_slot(struct active_request_slot *slot,if (!results) results = &results_buf; - err =run_one_slot (slot, results); + err =run_one_slot (slot, results,NULL );if (err !=HTTP_OK && err !=HTTP_REAUTH ) {struct strbuf msg =STRBUF_INIT ;
t/t5539-fetch-http-shallow.sh +30 -11
50 lines not on this page · raw message
t/t5540-http-push-webdav.sh +25 -1
@@ -68,12 +68,36 @@ test_expect_success 'push already up-to-date' ' git push ' +test_expect_success 'push to remote repository with packed refs and flakey server' ' + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + rm -rf packed-refs && + git update-ref refs/heads/master $ORIG_HEAD && + git --bare update-server-info) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/test_repo.git && + git push && + git remote set-url origin $HTTPD_URL/dumb/test_repo.git && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + test_expect_success 'push to remote repository with unpacked refs' ' (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && - rm packed-refs && + rm -rf packed-refs && + git update-ref refs/heads/master $ORIG_HEAD && + git --bare update-server-info) && + git push && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + +test_expect_success 'push to remote repository with unpacked refs and flakey server' ' + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + rm -rf packed-refs && git update-ref refs/heads/master $ORIG_HEAD && git --bare update-server-info) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/test_repo.git && git push && + git remote set-url origin $HTTPD_URL/dumb/test_repo.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && test $HEAD = $(git rev-parse --verify HEAD)) '
t/t5541-http-push-smart.sh +15 -0
@@ -77,6 +77,21 @@ test_expect_success 'push to remote repository (standard)' ' test $HEAD = $(git rev-parse --verify HEAD)) ' +test_expect_success 'push to remote repository (flakey server)' ' + cd "$ROOT_PATH"/test_repo_clone && + : >path5 && + git add path5 && + test_tick && + git commit -m path5 && + HEAD=$(git rev-parse --verify HEAD) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/smart/test_repo.git && + GIT_TRACE_CURL=true git push -v -v 2>err && + git remote set-url origin $HTTPD_URL/smart/test_repo.git && + grep "POST git-receive-pack ([0-9]* bytes)" err && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + test_expect_success 'push already up-to-date' ' git push '
t/t5550-http-fetch-dumb.sh +12 -0
@@ -176,6 +176,18 @@ test_expect_success 'fetch changes via http' ' test_cmp file clone/file ' +test_expect_success 'fetch changes via flakey http' ' + echo content >>file && + git commit -a -m three && + git push public && + (cd clone && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/repo.git && + git pull 2>../err && + git remote set-url origin $HTTPD_URL/dumb/repo.git) && + test_i18ngrep "got HTTP response 502" err && + test_cmp file clone/file +' + test_expect_success 'fetch changes via manual http-fetch' ' cp -R clone-tmpl clone2 &&
t/t5551-http-fetch-smart.sh +14 -0
@@ -45,6 +45,7 @@ test_expect_success 'clone http repository' ' EOF GIT_TRACE_CURL=true GIT_TEST_PROTOCOL_VERSION=0 \ git clone --quiet $HTTPD_URL/smart/repo.git clone 2>err && + cd clone && git config pull.rebase false && cd .. && test_cmp file clone/file && tr '\''\015'\'' Q <err | sed -e "
@@ -103,6 +104,19 @@ test_expect_success 'fetch changes via http' ' test_cmp file clone/file ' +test_expect_success 'fetch changes via flakey http' ' + echo content >>file && + git commit -a -m two && + git push public && + (cd clone && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/smart/repo.git && + git pull 2>../err && + git remote set-url origin $HTTPD_URL/smart/repo.git) && + test_cmp file clone/file && + test_i18ngrep "got HTTP response 502" err +' + + test_expect_success 'used upload-pack service' ' cat >exp <<-\EOF && GET /smart/repo.git/info/refs?service=git-upload-pack HTTP/1.1 200
t/t5601-clone.sh +5 -1
@@ -757,13 +757,17 @@ test_expect_success 'partial clone using HTTP' ' ' test_expect_success 'partial clone using HTTP with redirect' ' - _NONCE=`gen_nonce` && export _NONCE && + _NONCE=`gen_nonce` && curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" ' +test_expect_success 'partial clone with retry' ' + partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/`gen_nonce`/3/429/1/smart/server" 2>err && + test_i18ngrep "got HTTP response 429" err +' # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled.
Signature · 1 line
2.28.0.1011.ga647a8990f-goog
Re: [PATCH 3/3] http: automatically retry some requests
Hi Sean,
Quoted 11 lines · Sean McAllister
Some HTTP response codes indicate a server state that can support retrying the request rather than immediately erroring out. The server can also provide information about how long to wait before retries to via the Retry-After header. So check the server response and retry some reasonable number of times before erroring out to better accomodate transient errors.
Exiting immediately becomes irksome when pulling large multi-repo code bases such as Android or Chromium, as often the entire fetch operation has to be restarted from the beginning due to an error in one repo. If we can reduce how often that occurs, then it's a big win.
Makes a lot of sense to me.
Quoted 20 lines
Signed-off-by Sean McAllister <[email protected]>--- Documentation/config/http.txt | 5 ++ http.c | 121 +++++++++++++++++++++++++++++++++- http.h | 13 +++- remote-curl.c | 2 +- t/t5539-fetch-http-shallow.sh | 41 ++++++++---- t/t5540-http-push-webdav.sh | 26 +++++++- t/t5541-http-push-smart.sh | 15 +++++ t/t5550-http-fetch-dumb.sh | 12 ++++ t/t5551-http-fetch-smart.sh | 14 ++++ t/t5601-clone.sh | 6 +- 10 files changed, 237 insertions(+), 18 deletions(-)Documentation/config/http.txt +0
-0
Documentation/config/http.txt http.followRedirects:: · line 264
Quoted 5 lines
Documentation/config/http.txt +2
-0@@ -260,6 +260,11 @@ http.followRedirects:: the base for the follow-up requests, this is generally sufficient. The default is `initial`. +http.retryLimit:: + Some HTTP error codes (eg: 429,503) can reasonably be retried if
Please have a space after the comma so that it is not being mistaken for a 6-digit number. Also, aren't they called "status codes"? Not all of them indicate errors, after all.
Quoted 11 lines
Documentation/config/http.txt +3
-0+ they're encountered. This value configures the number of retry attempts + before giving up. The default retry limit is 3. + http.<url>.*:: Any of the http.* options above can be applied selectively to some URLs. For a config key to match a URL, each element of the config key ishttp.c +0
-0
http.c static const char *http_proxy_ssl_key; · line 96
Quoted 5 lines
http.c +2
-0@@ -92,6 +92,9 @@ static const char *http_proxy_ssl_key;static const char *http_proxy_ssl_ca_info;static struct credential proxy_cert_auth =CREDENTIAL_INIT ;static int proxy_ssl_cert_password_required; +static int http_retry_limit =3 ; +static int http_default_delay =2 ;
Should there be a config option for that? Also, it took me some time to find the code using this variable in order to find out what unit to use:
it is seconds (not microseconds, as I had expected). Maybe this can be documented in the variable name, or at least in a comment?
Quoted 12 lines
http.c +5
-0+static int http_max_delay =60 ; static struct { const char *name;@@ -219,6 +222,51 @@ size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)return nmemb; } + +/* return 1 for a retryable HTTP code, 0 otherwise */ +static int retryable_code (int code) +{
http.c size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf) · line 238
Quoted 10 lines
http.c +10
-0+switch (code) { +case 429 :/* fallthrough */ +case 502 :/* fallthrough */ +case 503 :/* fallthrough */ +case 504 :return 1 ; +default :return 0 ; + } +} + +size_t http_header_value(
In the part of Git's code with which I am familiar, we avoid trying to break the line after an opening parenthesis, instead preferring to break after a comma.
Also, shouldn't we make the return type ssize_t to allow for a negative value to indicate an error/missing header?
Quoted 6 lines
http.c +6
-0+ const struct strbuf headers, const char *header, char **value) +{ + size_t len = 0; +struct strbuf **lines, **line; +char *colon =NULL ; +
http.c size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf) · line 254
Quoted 10 lines
http.c +10
-0+ lines =strbuf_split (&headers,'\n' ); +for (line = lines; *line; line++) { +strbuf_trim (*line); + +/* find colon and null it out to 'split' string */ + colon =strchr ((*line)->buf,':' ); +if (colon) { + *colon ='\0' ; + +if (!strcasecmp (header, (*line)->buf)) {
If all we want is to find the given header, splitting lines seems to be a bit wasteful to me. We could instead search for the header directly:
const char *p = strcasestr(headers.buf, header), *eol;
size_t header_len = strlen(header);
while (p) {
if ((p != headers.buf && p[-1] != '\n') ||
p[header_len] != ':') {
p = strcasestr(p + header_len, header);
continue;
}
p += header_len + 1;
while (isspace(*p) && *p != '\n')
p++;
eol = strchrnul(p, '\n');
len = eol - p;
*value = xstrndup(p, len);
return len;
}Quoted 49 lines
http.c +23
-1+/* move past colon and skip whitespace */ + colon++; +while (*colon &&isspace (*colon)) colon++; + *value =xstrdup (colon); + len =strlen (*value); + goto done; + } + } + } + +done: +strbuf_list_free (lines); +return len; +} +static void closedown_active_slot (struct active_request_slot *slot) { active_requests--;@@ -452,6 +500,11 @@ static int http_options(const char *var, const char *value, void *cb)return 0 ; } +if (!strcmp ("http.retrylimit" , var)) { + http_retry_limit =git_config_int (var, value); +return 0 ; + } +/* Fall back on the default ones */ return git_default_config (var, value, cb); }@@ -1668,7 +1721,7 @@ static int handle_curl_result(struct slot_results *results) }int run_one_slot (struct active_request_slot *slot, -struct slot_results *results) +struct slot_results *results,int *http_code) { slot->results = results;if (!start_active_slot (slot)) {@@ -1678,6 +1731,8 @@ int run_one_slot(struct active_request_slot *slot, }run_active_slot (slot); +if (http_code) + *http_code = results->http_code;return handle_curl_result (results); }
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1963
Quoted 6 lines
http.c +3
-0@@ -1903,20 +1958,55 @@ static void http_opt_request_remainder(CURL *curl, off_t pos)#define HTTP_REQUEST_STRBUF 0#define HTTP_REQUEST_FILE 1 +/* check for a retry-after header in the given headers string, if found, then +honor it, otherwise do an exponential backoff up to the max on the current delay +*/
Multi-line comments should be of this form:
/*
* Check for a retry-after header in the given headers string, if
* found, then honor it, otherwise do an exponential backoff up to
* the maximum on the current delay.
*/
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1964
Quoted 1 line
http.c +1
-0+static int http_retry_after (const struct strbuf headers,int cur_delay) {
For functions, the initial opening curly bracket goes on its own line.
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1967
Quoted 3 lines
http.c +3
-0+int len, delay; +char *end; +char *value;
Why not declare char *end, *value;, just like len and delay are declared on the same line?
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1972
Quoted 5 lines
http.c +5
-0+ + len =http_header_value (headers,"retry-after" , &value); +if (len) { + delay =strtol (value, &end,0 ); +if (*value && *end ==0 && delay >=0 ) {
Better: *end == '\0'
And why *value here? We already called strtol() on it.
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1974
Quoted 2 lines
http.c +2
-0+if (delay > http_max_delay) { + die(Q_(
Let's not end the line in an opening parenthesis. Instead, use C's string continuation like so:
die(Q_("server requested retry after %d second,"
" which is longer than max allowed\n", "server requested retry after %d " "seconds, which is longer than max " "allowed\n", delay), delay);
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1978
Quoted 4 lines
http.c +4
-0+"server requested retry after %d second, which is longer than max allowed\n" , + "server requested retry after %d seconds, which is longer than max allowed\n", delay), delay); + } +free (value);
value is not actually used after that strtol() call above, so let's release it right then and there.
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1982
Quoted 4 lines
http.c +4
-0+return delay; + } +free (value); + }
If the header was found, but for some reason had an empty value, we're leaking value here.
Quoted 40 lines
http.c +15
-1+ + cur_delay *=2 ; +return cur_delay >= http_max_delay ? http_max_delay : cur_delay; +} +static int http_request (const char *url,void *result,int target,const struct http_get_options *options) {struct active_request_slot *slot;struct slot_results results; -struct curl_slist *headers =http_copy_default_headers (); +struct curl_slist *headers;struct strbuf buf =STRBUF_INIT ; +struct strbuf result_headers =STRBUF_INIT ;const char *accept_language;int ret; +int retry_cnt =0 ; +int retry_delay = http_default_delay; +int http_code; +retry: slot =get_active_slot ();curl_easy_setopt (slot->curl,CURLOPT_HTTPGET ,1 ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA , &result_headers); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION , fwrite_buffer); +if (result ==NULL ) {curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,1 ); }else {@@ -1936,6 +2026,7 @@ static int http_request(const char *url, accept_language =get_accept_language (); + headers =http_copy_default_headers ();if (accept_language) headers =curl_slist_append (headers, accept_language);
http.c static int http_request(const char *url, · line 2059
Quoted 9 lines
http.c +5
-1@@ -1961,7 +2052,31 @@ static int http_request(const char *url,curl_easy_setopt (slot->curl,CURLOPT_ENCODING ,"" );curl_easy_setopt (slot->curl,CURLOPT_FAILONERROR ,0 ); - ret =run_one_slot (slot, &results); + http_code =0 ; + ret =run_one_slot (slot, &results, &http_code); + +if (ret !=HTTP_OK ) { +if (retryable_code (http_code) && (retry_cnt < http_retry_limit)) {
The parentheses around the second condition should be dropped.
http.c static int http_request(const char *url, · line 2062
Quoted 3 lines
http.c +3
-0+ retry_cnt++; + retry_delay =http_retry_after (result_headers, retry_delay); + fprintf(stderr,
Should this be a warning() instead? I see 5 instances in http.c that use fprintf(stderr, ...), but 12 that use warning(), making me believe that at least some of those 5 instances should call warning() instead, too.
http.c static int http_request(const char *url, · line 2069
Quoted 7 lines
http.c +7
-0+Q_ ("got HTTP response %d, retrying after %d second (%d/%d)\n" , +"got HTTP response %d, retrying after %d seconds (%d/%d)\n" , + retry_delay), + http_code, retry_delay, retry_cnt, http_retry_limit); +sleep (retry_delay); + +// remove header data fields since not all slots will use them
No C++-style comments, please: use /* ... */ instead.
http.c static int http_request(const char *url, · line 2077
Quoted 8 lines
http.c +8
-0+curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA ,NULL ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION ,NULL ); + + goto retry; + } + } + +// remove header data fields since not all slots will use them
No C++-style comments, please.
http.c static int http_request(const char *url, · line 2079
Quoted 2 lines
http.c +2
-0+curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA ,NULL ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION ,NULL );
Shouldn't we just perform this assignment before the if (ret != HTTP_OK) condition? I do not see anything inside that block that needs it, therefore this could be DRY'd up.
Quoted 11 lines
http.c +0
-0if (options && options->content_type) {struct strbuf raw =STRBUF_INIT ;http.h +0
-0@@ -86,6 +86,17 @@ size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf);curlioerr ioctl_buffer (CURL *handle,int cmd,void *clientp);#endif
http.h size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf); · line 98
Quoted 10 lines
http.h +10
-0+/* +* Query the value of an HTTP header. +* +* If the header is found, then a newly allocate string is returned through +* the value parameter, and the length is returned. +* +* If not found, returns 0 +*/ +size_t http_header_value ( +const struct strbuf headers,const char *header,char **value);
Do we really need to export this function? It could stay file-local, at least for now (i.e. be defined static inside http.c), no?
Quoted 46 lines
http.h +2
-1+/* Slot lifecycle functions */ struct active_request_slot *get_active_slot (void );int start_active_slot (struct active_request_slot *slot);@@ -99,7 +110,7 @@ void finish_all_active_slots(void);* */ int run_one_slot (struct active_request_slot *slot, -struct slot_results *results); +struct slot_results *results,int *http_code);#ifdef USE_CURL_MULTI void fill_active_slots (void );remote-curl.c +1
-1@@ -805,7 +805,7 @@ static int run_slot(struct active_request_slot *slot,if (!results) results = &results_buf; - err =run_one_slot (slot, results); + err =run_one_slot (slot, results,NULL );if (err !=HTTP_OK && err !=HTTP_REAUTH ) {struct strbuf msg =STRBUF_INIT ;t/t5539-fetch-http-shallow.sh +1
-11@@ -30,20 +30,39 @@ test_expect_success 'clone http repository' ' git clone --bare --no-local shallow "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" && git clone $HTTPD_URL/smart/repo.git clone && ( - cd clone && - git fsck && - git log --format=%s origin/master >actual && - cat <<EOF >expect && -7 -6 -5 -4 -3 -EOF - test_cmp expect actual + cd clone &&
t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 43
Quoted 10 lines
t/t5539-fetch-http-shallow.sh +10
-0+ git fsck && + git log --format=%s origin/master >actual && + cat <<-\EOF >expect && + 7 + 6 + 5 + 4 + 3 + EOF + test_cmp expect actual
This just changes the indentation, right?
I _guess_ this is a good change, but it should live in its own patch.
t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 48
Quoted 5 lines
t/t5539-fetch-http-shallow.sh +2
-0) ' +test_expect_success 'clone http repository with flaky http' ' + rm -rf clone &&
Let's consistently use horizontal tab characters for indentation. (There are more instances of lines indented by spaces below.)
t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 49
Quoted 1 line
t/t5539-fetch-http-shallow.sh +1
-0+ git clone $HTTPD_URL/error_ntime/`gen_nonce`/3/429/1/smart/repo.git clone 2>err &&
Let's use $(gen_nonce). Also: where is the gen_nonce defined? I do not see the definition in this patch (but it could be 1/3, which for some reason did not make it to the mailing list:
https://lore.kernel.org/git/[email protected]/).
Another suggestion: rather than deleting clone/, use a separate directory to clone into, say, flaky/. That will make it easier to debug when the entire "trash" directory is tar'ed up in a failed CI build, for example.
Quoted 115 lines
t/t5539-fetch-http-shallow.sh +16
-0+ ( + cd clone && + git fsck && + git log --format=%s origin/master >actual && + cat <<-\EOF >expect && + 7 + 6 + 5 + 4 + 3 + EOF + test_cmp expect actual + ) && + test_i18ngrep "got HTTP response 429" err +' + # This test is tricky. We need large enough "have"s that fetch-pack # will put pkt-flush in between. Then we need a "have" the server # does not have, it'll send "ACK %s ready"t/t5540-http-push-webdav.sh +25
-1@@ -68,12 +68,36 @@ test_expect_success 'push already up-to-date' ' git push ' +test_expect_success 'push to remote repository with packed refs and flakey server' ' + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + rm -rf packed-refs && + git update-ref refs/heads/master $ORIG_HEAD && + git --bare update-server-info) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/test_repo.git && + git push && + git remote set-url origin $HTTPD_URL/dumb/test_repo.git && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + test_expect_success 'push to remote repository with unpacked refs' ' (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && - rm packed-refs && + rm -rf packed-refs && + git update-ref refs/heads/master $ORIG_HEAD && + git --bare update-server-info) && + git push && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + +test_expect_success 'push to remote repository with unpacked refs and flakey server' ' + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + rm -rf packed-refs && git update-ref refs/heads/master $ORIG_HEAD && git --bare update-server-info) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/test_repo.git && git push && + git remote set-url origin $HTTPD_URL/dumb/test_repo.git && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && test $HEAD = $(git rev-parse --verify HEAD)) 't/t5541-http-push-smart.sh +15
-0@@ -77,6 +77,21 @@ test_expect_success 'push to remote repository (standard)' ' test $HEAD = $(git rev-parse --verify HEAD)) ' +test_expect_success 'push to remote repository (flakey server)' ' + cd "$ROOT_PATH"/test_repo_clone && + : >path5 && + git add path5 && + test_tick && + git commit -m path5 && + HEAD=$(git rev-parse --verify HEAD) && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/smart/test_repo.git && + GIT_TRACE_CURL=true git push -v -v 2>err && + git remote set-url origin $HTTPD_URL/smart/test_repo.git && + grep "POST git-receive-pack ([0-9]* bytes)" err && + (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && + test $HEAD = $(git rev-parse --verify HEAD)) +' + test_expect_success 'push already up-to-date' ' git push 't/t5550-http-fetch-dumb.sh +12
-0@@ -176,6 +176,18 @@ test_expect_success 'fetch changes via http' ' test_cmp file clone/file ' +test_expect_success 'fetch changes via flakey http' ' + echo content >>file && + git commit -a -m three && + git push public && + (cd clone && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/dumb/repo.git && + git pull 2>../err && + git remote set-url origin $HTTPD_URL/dumb/repo.git) && + test_i18ngrep "got HTTP response 502" err && + test_cmp file clone/file +' + test_expect_success 'fetch changes via manual http-fetch' ' cp -R clone-tmpl clone2 &&t/t5551-http-fetch-smart.sh +0
-0
t/t5551-http-fetch-smart.sh test_expect_success 'clone http repository' ' · line 48
Quoted 4 lines
t/t5551-http-fetch-smart.sh +1
-0@@ -45,6 +45,7 @@ test_expect_success 'clone http repository' ' EOF GIT_TRACE_CURL=true GIT_TEST_PROTOCOL_VERSION=0 \ git clone --quiet $HTTPD_URL/smart/repo.git clone 2>err && + cd clone && git config pull.rebase false && cd .. &&
Better: test_config -C clone pull.rebase false
Quoted 33 lines
t/t5551-http-fetch-smart.sh +13
-0test_cmp file clone/file && tr '\''\015'\'' Q <err | sed -e "@@ -103,6 +104,19 @@ test_expect_success 'fetch changes via http' ' test_cmp file clone/file ' +test_expect_success 'fetch changes via flakey http' ' + echo content >>file && + git commit -a -m two && + git push public && + (cd clone && + git remote set-url origin $HTTPD_URL/error_ntime/`gen_nonce`/3/502/1/smart/repo.git && + git pull 2>../err && + git remote set-url origin $HTTPD_URL/smart/repo.git) && + test_cmp file clone/file && + test_i18ngrep "got HTTP response 502" err +' + + test_expect_success 'used upload-pack service' ' cat >exp <<-\EOF && GET /smart/repo.git/info/refs?service=git-upload-pack HTTP/1.1 200t/t5601-clone.sh +1
-1@@ -757,13 +757,17 @@ test_expect_success 'partial clone using HTTP' ' ' test_expect_success 'partial clone using HTTP with redirect' ' - _NONCE=`gen_nonce` && export _NONCE && + _NONCE=`gen_nonce` &&
t/t5601-clone.sh test_expect_success 'partial clone using HTTP' ' · line 770
Quoted 10 lines
t/t5601-clone.sh +4
-0curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" ' +test_expect_success 'partial clone with retry' ' + partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/`gen_nonce`/3/429/1/smart/server" 2>err && + test_i18ngrep "got HTTP response 429" err +'
I wonder whether it is really necessary to add _that_ many test cases. The test suite already takes so long to run that we had cases where contributors simply did not run it before sending their contributions.
In this instance, I would think that it would be plenty sufficient to have a single new test case that exercizes the added code path (and verifies that it can see the message).
Thanks,
Dscho
Quoted 7 lines
t/t5601-clone.sh +0
-1# DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled. --2.28.0.1011.ga647a8990f-goog
Re: [PATCH 3/3] http: automatically retry some requests
Quoted 18 lines · Johannes Schindelin
Hi Sean,
Quoted 11 lines · Sean McAllister
Some HTTP response codes indicate a server state that can support retrying the request rather than immediately erroring out. The server can also provide information about how long to wait before retries to via the Retry-After header. So check the server response and retry some reasonable number of times before erroring out to better accomodate transient errors.
Exiting immediately becomes irksome when pulling large multi-repo code bases such as Android or Chromium, as often the entire fetch operation has to be restarted from the beginning due to an error in one repo. If we can reduce how often that occurs, then it's a big win.
Makes a lot of sense to me.
...
Quoted 7 lines
Documentation/config/http.txt http.followRedirects:: · line 264
Quoted 2 lines
Documentation/config/http.txt +2
-0+http.retryLimit:: + Some HTTP error codes (eg: 429,503) can reasonably be retried ifPlease have a space after the comma so that it is not being mistaken for a 6-digit number. Also, aren't they called "status codes"? Not all of them indicate errors, after all.
...
I've read your comments and agree to them all. Thanks for a detailed and excellent review.
Re: [PATCH 3/3] http: automatically retry some requests
In reply to Johannes SchindelinQuoted 7 lines
Documentation/config/http.txt http.followRedirects:: · line 264
Quoted 3 lines
Documentation/config/http.txt +2
-0+http.retryLimit:: + Some HTTP error codes (eg: 429,503) can reasonably be retried ifPlease have a space after the comma so that it is not being mistaken for a 6-digit number. Also, aren't they called "status codes"? Not all of them indicate errors, after all.
Done for v2, good point on the nomenclature.
Quoted 7 lines
Quoted 7 lines
http.c +0
-0@@ -92,6 +92,9 @@ static const char *http_proxy_ssl_key;static const char *http_proxy_ssl_ca_info;static struct credential proxy_cert_auth =CREDENTIAL_INIT ;
Quoted 9 lines
http.c static const char *http_proxy_ssl_key; · line 96
Quoted 3 lines
http.c +2
-0static int proxy_ssl_cert_password_required; +static int http_retry_limit =3 ; +static int http_default_delay =2 ;Should there be a config option for that? Also, it took me some time to find the code using this variable in order to find out what unit to use:
it is seconds (not microseconds, as I had expected). Maybe this can be documented in the variable name, or at least in a comment?
Junio tossed that out during our private review and I think we decided to just leave them as non-const so we have that option going forward. I'm not sure there's
a strong story for configuring the default delay.
I went through and changed all the _delay variables to be more explicit they're in seconds.
Quoted 23 lines
Quoted 8 lines
http.c +4
-0@@ -219,6 +222,51 @@ size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)return nmemb; } + +/* return 1 for a retryable HTTP code, 0 otherwise */ +static int retryable_code (int code) +{http.c size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf) · line 238
Quoted 10 lines
http.c +10
-0+switch (code) { +case 429 :/* fallthrough */ +case 502 :/* fallthrough */ +case 503 :/* fallthrough */ +case 504 :return 1 ; +default :return 0 ; + } +} + +size_t http_header_value(In the part of Git's code with which I am familiar, we avoid trying to break the line after an opening parenthesis, instead preferring to break after a comma.
Quoted 2 lines
Also, shouldn't we make the return type
ssize_tto allow for a negative value to indicate an error/missing header?
I return zero to indicate no header found (or zero-length value), so this should never return a negative value. One can check value to see if a string was allocated
for a zero-length header.
Quoted 31 lines
Quoted 7 lines
http.c +6
-0+ const struct strbuf headers, const char *header, char **value) +{ + size_t len = 0; +struct strbuf **lines, **line; +char *colon =NULL ; +http.c size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf) · line 254
Quoted 10 lines
http.c +10
-0+ lines =strbuf_split (&headers,'\n' ); +for (line = lines; *line; line++) { +strbuf_trim (*line); + +/* find colon and null it out to 'split' string */ + colon =strchr ((*line)->buf,':' ); +if (colon) { + *colon ='\0' ; + +if (!strcasecmp (header, (*line)->buf)) {If all we want is to find the given header, splitting lines seems to be a bit wasteful to me. We could instead search for the header directly:
const char *p = strcasestr(headers.buf, header), *eol; size_t header_len = strlen(header); while (p) { if ((p != headers.buf && p[-1] != '\n') || p[header_len] != ':') { p = strcasestr(p + header_len, header); continue; }
Quoted 9 lines
p += header_len + 1;
while (isspace(*p) && *p != '\n')
p++;
eol = strchrnul(p, '\n');
len = eol - p;
*value = xstrndup(p, len);
return len;
}I've been writing a lot of python code lately =D So splitting into lines was a natural paradigm for me. You're right, I like yours more. I've refactored it to be closer to that. Little bit of fiddling to deal with header whitespace properly, but it's pretty close.
I also modified to just return the value pointer directly, then it's clear when we get a zero-length header or don't find it completely, and it fixes the leak issue below.
Quoted 10 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1963
Quoted 7 lines
http.c +3
-0@@ -1903,20 +1958,55 @@ static void http_opt_request_remainder(CURL *curl, off_t pos)#define HTTP_REQUEST_STRBUF 0#define HTTP_REQUEST_FILE 1 +/* check for a retry-after header in the given headers string, if found, then +honor it, otherwise do an exponential backoff up to the max on the current delay +*/ Multi-line comments should be of this form:
Quoted 6 lines
/*
* Check for a retry-after header in the given headers string, if
* found, then honor it, otherwise do an exponential backoff up to
* the maximum on the current delay.
*/
Done.
Quoted 4 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1964
Quoted 1 line
http.c +1
-0+static int http_retry_after (const struct strbuf headers,int cur_delay) {For functions, the initial opening curly bracket goes on its own line.
Done.
Quoted 7 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1967
Quoted 3 lines
http.c +3
-0+int len, delay; +char *end; +char *value;Why not declare
char *end, *value;, just likelenanddelayare declared on the same line?
Done.
Quoted 10 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1972
Quoted 5 lines
http.c +5
-0+ + len =http_header_value (headers,"retry-after" , &value); +if (len) { + delay =strtol (value, &end,0 ); +if (*value && *end ==0 && delay >=0 ) {Better:
*end == '\0'And why
*valuehere? We already calledstrtol()on it.
Fixed, and I check value per the man page on strtol:
Quoted 1 line
In particular, if *nptr is not '\0' but **endptr is '\0' on
return, the entire string is valid.
I wanted to make sure I converted the entire header value, so this seemed correct?
Quoted 12 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1974
Quoted 2 lines
http.c +2
-0+if (delay > http_max_delay) { + die(Q_(Let's not end the line in an opening parenthesis. Instead, use C's string continuation like so:
die(Q_("server requested retry after %d second,"
" which is longer than max allowed\n", "server requested retry after %d " "seconds, which is longer than max " "allowed\n", delay), delay);
Done.
Quoted 8 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1978
Quoted 4 lines
http.c +4
-0+"server requested retry after %d second, which is longer than max allowed\n" , + "server requested retry after %d seconds, which is longer than max allowed\n", delay), delay); + } +free (value);
valueis not actually used after thatstrtol()call above, so let's release it right then and there.
Good call, done.
Quoted 8 lines
http.c static void http_opt_request_remainder(CURL *curl, off_t pos) · line 1982
Quoted 4 lines
http.c +4
-0+return delay; + } +free (value); + }If the header was found, but for some reason had an empty value, we're leaking
valuehere.
I return the pointer directly and check that now, so if it's allocated, we'll always call free, even if it's an empty string.
Quoted 43 lines
Quoted 43 lines
http.c +15
-1+ + cur_delay *=2 ; +return cur_delay >= http_max_delay ? http_max_delay : cur_delay; +} +static int http_request (const char *url,void *result,int target,const struct http_get_options *options) {struct active_request_slot *slot;struct slot_results results; -struct curl_slist *headers =http_copy_default_headers (); +struct curl_slist *headers;struct strbuf buf =STRBUF_INIT ; +struct strbuf result_headers =STRBUF_INIT ;const char *accept_language;int ret; +int retry_cnt =0 ; +int retry_delay = http_default_delay; +int http_code; +retry: slot =get_active_slot ();curl_easy_setopt (slot->curl,CURLOPT_HTTPGET ,1 ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA , &result_headers); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION , fwrite_buffer); +if (result ==NULL ) {curl_easy_setopt (slot->curl,CURLOPT_NOBODY ,1 ); }else {@@ -1936,6 +2026,7 @@ static int http_request(const char *url, accept_language =get_accept_language (); + headers =http_copy_default_headers ();if (accept_language) headers =curl_slist_append (headers, accept_language);@@ -1961,7 +2052,31 @@ static int http_request(const char *url,curl_easy_setopt (slot->curl,CURLOPT_ENCODING ,"" );curl_easy_setopt (slot->curl,CURLOPT_FAILONERROR ,0 );
Quoted 9 lines
http.c static int http_request(const char *url, · line 2059
Quoted 6 lines
http.c +5
-1- ret =run_one_slot (slot, &results); + http_code =0 ; + ret =run_one_slot (slot, &results, &http_code); + +if (ret !=HTTP_OK ) { +if (retryable_code (http_code) && (retry_cnt < http_retry_limit)) {The parentheses around the second condition should be dropped.
Done.
Quoted 9 lines
http.c static int http_request(const char *url, · line 2062
Quoted 3 lines
http.c +3
-0+ retry_cnt++; + retry_delay =http_retry_after (result_headers, retry_delay); + fprintf(stderr,Should this be a
warning()instead? I see 5 instances inhttp.cthat usefprintf(stderr, ...), but 12 that usewarning(), making me believe that at least some of those 5 instances should callwarning()instead, too.
At your discretion. I'm not familiar with the logging framework. Only one of those 5 fprintf
is in my patch, so I changed that one to use warning().
Quoted 10 lines
http.c static int http_request(const char *url, · line 2069
Quoted 7 lines
http.c +7
-0+Q_ ("got HTTP response %d, retrying after %d second (%d/%d)\n" , +"got HTTP response %d, retrying after %d seconds (%d/%d)\n" , + retry_delay), + http_code, retry_delay, retry_cnt, http_retry_limit); +sleep (retry_delay); + +// remove header data fields since not all slots will use them No C++-style comments, please: use /* ... */ instead.
Thought I got them all. Done.
Quoted 11 lines
http.c static int http_request(const char *url, · line 2077
Quoted 8 lines
http.c +8
-0+curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA ,NULL ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION ,NULL ); + + goto retry; + } + } + +// remove header data fields since not all slots will use them No C++-style comments, please.
Done.
Quoted 7 lines
http.c static int http_request(const char *url, · line 2079
Quoted 2 lines
http.c +2
-0+curl_easy_setopt (slot->curl,CURLOPT_HEADERDATA ,NULL ); +curl_easy_setopt (slot->curl,CURLOPT_HEADERFUNCTION ,NULL );Shouldn't we just perform this assignment before the
if (ret != HTTP_OK)condition? I do not see anything inside that block that needs it, therefore this could be DRY'd up.
Excellent idea, done.
Quoted 5 lines
Quoted 5 lines
http.c +5
-0+/* +* Query the value of an HTTP header. +* +* If the header is found, then a newly allocate string is returned through +* the value parameter, and the length is returned.
Quoted 9 lines
http.c
Quoted 5 lines
+ *
+ * If not found, returns 0
+ */http.c +2
-0+size_t http_header_value ( +const struct strbuf headers,const char *header,char **value);Do we really need to export this function? It could stay file-local, at least for now (i.e. be defined
staticinsidehttp.c), no?
I originally thought I'd need it elsewhere, so a big of a relic there. I made it static for now.
Quoted 31 lines
Quoted 18 lines
t/t5539-fetch-http-shallow.sh +1
-11@@ -30,20 +30,39 @@ test_expect_success 'clone http repository' ' git clone --bare --no-local shallow "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" && git clone $HTTPD_URL/smart/repo.git clone && ( - cd clone && - git fsck && - git log --format=%s origin/master >actual && - cat <<EOF >expect && -7 -6 -5 -4 -3 -EOF - test_cmp expect actual + cd clone &&t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 43
Quoted 10 lines
t/t5539-fetch-http-shallow.sh +10
-0+ git fsck && + git log --format=%s origin/master >actual && + cat <<-\EOF >expect && + 7 + 6 + 5 + 4 + 3 + EOF + test_cmp expect actualThis just changes the indentation, right?
Quoted 2 lines
I _guess_ this is a good change, but it should live in its own patch.
This was in combination with my adding a new test, we cleaned up the whitespace too per Junio rather than copy-and-pasting ill formatted code.
It's OBE now because this change has been removed in v2.
Quoted 6 lines
t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 48
Quoted 2 lines
t/t5539-fetch-http-shallow.sh +2
-0+test_expect_success 'clone http repository with flaky http' ' + rm -rf clone &&Let's consistently use horizontal tab characters for indentation. (There are more instances of lines indented by spaces below.)
*sigh* thought I got them all, fixed.
Quoted 7 lines
t/t5539-fetch-http-shallow.sh test_expect_success 'clone http repository' ' · line 49
Quoted 1 line
t/t5539-fetch-http-shallow.sh +1
-0+ git clone $HTTPD_URL/error_ntime/`gen_nonce`/3/429/1/smart/repo.git clone 2>err &&Let's use
$(gen_nonce). Also: where is thegen_noncedefined? I do not see the definition in this patch (but it could be 1/3, which for some reason did not make it to the mailing list:
https://lore.kernel.org/git/[email protected]/).
Done, and it is indeed in 1/3 which got caught by the spam filter (should be available now).
Quoted 5 lines
Another suggestion: rather than deleting
clone/, use a separate directory to clone into, say,flaky/. That will make it easier to debug when the entire "trash" directory is tar'ed up in a failed CI build, for example.
Done.
Quoted 12 lines
t/t5551-http-fetch-smart.sh test_expect_success 'clone http repository' ' · line 48
Quoted 9 lines
t/t5551-http-fetch-smart.sh +1
-0@@ -45,6 +45,7 @@ test_expect_success 'clone http repository' ' EOF GIT_TRACE_CURL=true GIT_TEST_PROTOCOL_VERSION=0 \ git clone --quiet $HTTPD_URL/smart/repo.git clone 2>err && + cd clone && git config pull.rebase false && cd .. &&Better:
test_config-C clone pull.rebase false
Done, but also OBE by removing tests.
Quoted 9 lines
I wonder whether it is really necessary to add _that_ many test cases. The test suite already takes so long to run that we had cases where contributors simply did not run it before sending their contributions.
In this instance, I would think that it would be plenty sufficient to have a single new test case that exercizes the added code path (and verifies that it can see the message).
Done, down to a single test (clone) that should exercise everything.
Quoted 10 lines
Thanks,
DschoQuoted 7 lines
t/t5551-http-fetch-smart.sh +0
-1# DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled. --2.28.0.1011.ga647a8990f-goog
Re: [PATCH 3/3] http: automatically retry some requests
Quoted 9 lines · Sean McAllister
Quoted 7 lines
http.c static const char *http_proxy_ssl_key; · line 96
Quoted 2 lines
http.c +2
-0+static int http_retry_limit =3 ; +static int http_default_delay =2 ;Should there be a config option for that? Also, it took me some time to find the code using this variable in order to find out what unit to use:
it is seconds (not microseconds, as I had expected). Maybe this can be documented in the variable name, or at least in a comment?Junio tossed that out during our private review and I think we decided to just
Needs clarification. Here "that" in "tossed that out" only refers to "static int const http_retry_limit = 3" and friends and nothing else. There weren't any discussion on units or comments. I did mention that it is an obvious future possibility to make these configurable and that was why I suggested to "toss out" the const.
It seems we'll see names with "seconds" in them somewhere, which is good.
Quoted 4 lines
I've been writing a lot of python code lately =D So splitting into lines was a natural paradigm for me. You're right, I like yours more. I've refactored it to be closer to that. Little bit of fiddling to deal with header whitespace properly, but it's pretty close.
Good. I personally think strbuf_split() is a mistaken API whose use needs to be killed, so it makes me happy to see one new callsite we didn't have to add ;-)
Thanks.
PATCH 1/1 remote-curl: add testing for intelligent retry for HTTP
In reply to a message not in the archiveHTTP servers can sometimes throw errors, but that doesn't mean we should give up. If we have an error condition that we can reasonably retry on, then we should.
This change is tricky because it requires a new CGI script to test as we need to be able to instruct the server to throw an error(s) before succeeding. We do this by encoding an error code and optional value for Retry-After into the URL, followed by the real endpoint:
/error_ntime/dc724af1/<N>/429/10/smart/server
This is a bit hacky, but really the best we can do since HTTP is fundamentally stateless. The URL is uniquefied by a nonce and we leave a breadcrumb on disk so all accesses after the first <N> redirect to the appropriate endpoint.
--- t/lib-httpd.sh | 6 +++ t/lib-httpd/apache.conf | 9 +++++ t/lib-httpd/error-ntime.sh | 79 ++++++++++++++++++++++++++++++++++++++ t/t5601-clone.sh | 9 +++++ 4 files changed, 103 insertions(+) create mode 100755 t/lib-httpd/error-ntime.sh
t/lib-httpd.sh +6 -0
@@ -132,6 +132,7 @@ prepare_httpd() { install_script incomplete-length-upload-pack-v2-http.sh install_script incomplete-body-upload-pack-v2-http.sh install_script broken-smart-http.sh + install_script error-ntime.sh install_script error-smart-http.sh install_script error.sh install_script apply-one-time-perl.sh
@@ -308,3 +309,8 @@ check_access_log() { test_cmp "$1" access.log.stripped fi } + +# generate a random 12 digit string +gen_nonce() { + test_copy_bytes 12 < /dev/urandom | tr -dc A-Za-z0-9 +}
t/lib-httpd/apache.conf +9 -0
@@ -117,6 +117,12 @@ Alias /auth/dumb/ www/auth/dumb/ SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH} SetEnv GIT_HTTP_EXPORT_ALL </LocationMatch> + +# This may be suffixed with any path for redirection, so it should come before +# any of the other aliases, particularly the /smart_*[^/]*/(.*) alias as that can +# match a lot of URLs +ScriptAlias /error_ntime/ error-ntime.sh/ + ScriptAlias /smart/incomplete_length/git-upload-pack incomplete-length-upload-pack-v2-http.sh/ ScriptAlias /smart/incomplete_body/git-upload-pack incomplete-body-upload-pack-v2-http.sh/ ScriptAliasMatch /error_git_upload_pack/(.*)/git-upload-pack error.sh/
@@ -137,6 +143,9 @@ ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1 <Files broken-smart-http.sh> Options ExecCGI </Files> +<Files error-ntime.sh> + Options ExecCGI +</Files> <Files error-smart-http.sh> Options ExecCGI </Files>
t/lib-httpd/error-ntime.sh new file +79 -0
79 lines not on this page · raw message
t/t5601-clone.sh +9 -0
@@ -756,6 +756,15 @@ test_expect_success 'partial clone using HTTP' ' partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server" ' +test_expect_success 'partial clone using HTTP with redirect' ' + _NONCE=`gen_nonce` && export _NONCE && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" +' + + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled.
Signature · 1 line
2.28.0.1011.ga647a8990f-goog
Re: [PATCH] remote-curl: add testing for intelligent retry for HTTP
t/lib-httpd.sh check_access_log() { · line 316
Quoted 4 lines · Sean McAllister
t/lib-httpd.sh +4
-0+# generate a random 12 digit string +gen_nonce() { + test_copy_bytes 12 < /dev/urandom | tr -dc A-Za-z0-9 +}
What is the randomness requirement of this application? IOW, what breaks if we just change this to "echo 0123456789ab"?
Or "date | git hash-object --stdin" for that matter?
We'd want to make our tests more predictiable, not less.
Quoted 25 lines
t/lib-httpd/error-ntime.sh new file +19
-0@@ -0,0 +1,79 @@ +#!/bin/sh + +# Script to simulate a transient error code with Retry-After header set. +# +# PATH_INFO must be of the form /<nonce>/<times>/<retcode>/<retry-after>/<path> +# (eg: /dc724af1/3/429/10/some/url) +# +# The <nonce> value uniquely identifies the URL, since we're simulating +# a stateful operation using a stateless protocol, we need a way to "namespace" +# URLs so that they don't step on each other. +# +# The first <times> times this endpoint is called, it will return the given +# <retcode>, and if the <retry-after> is non-negative, it will set the +# Retry-After head to that value. +# +# Subsequent calls will return a 302 redirect to <path>. +# +# Supported error codes are 429,502,503, and 504 +
t/lib-httpd/error-ntime.sh line 29
Quoted 10 lines
t/lib-httpd/error-ntime.sh new file +10
-0+print_status() { + if [ "$1" -eq "302" ]; then printf "Status: 302 Found\n" + elif [ "$1" -eq "429" ]; then printf "Status: 429 Too Many Requests\n" + elif [ "$1" -eq "502" ]; then printf "Status: 502 Bad Gateway\n" + elif [ "$1" -eq "503" ]; then printf "Status: 503 Service Unavailable\n" + elif [ "$1" -eq "504" ]; then printf "Status: 504 Gateway Timeout\n" + else + printf "Status: 500 Internal Server Error\n" + fi + printf "Content-Type: text/plain\n"
Style????? (I won't repeat this comment for the rest of this script)
I briefly wondered "oh, are t/lib-httpd/* scripts excempt from the coding guidelines?" but a quick look at them tells me that that is not the case.
t/lib-httpd/error-ntime.sh line 40
Quoted 11 lines
t/lib-httpd/error-ntime.sh new file +11
-0+} + +# read in path split into cmoponents +IFS='/' +tokens=($PATH_INFO) + +# break out code/retry parts of path +nonce=${tokens[1]} +times=${tokens[2]} +code=${tokens[3]} +retry=${tokens[4]}
You said /bin/sh upfront. Don't use non-POSIX shell arrays.
Quoted 44 lines
t/lib-httpd/error-ntime.sh new file +39
-0+ +# get redirect path +cnt=0 +path="" +for ((ii=0; ii < ${#PATH_INFO}; ii++)); do + if [ "${PATH_INFO:${ii}:1}" == "/" ]; then + let cnt=${cnt}+1 + fi + if [ "${cnt}" -eq 5 ]; then + path="${PATH_INFO:${ii}}" + break + fi +done + +# leave a cookie for this request/retry count +state_file="request_${REMOTE_ADDR}_${nonce}_${times}_${code}_${retry}" + +if [ ! -f "$state_file" ]; then + echo 0 > "$state_file" +fi + + +read cnt < "$state_file" +if [ "$cnt" -lt "$times" ]; then + let cnt=cnt+1 + echo "$cnt" > "$state_file" + + # return error + print_status "$code" + if [ "$retry" -ge "0" ]; then + printf "Retry-After: %s\n" "$retry" + fi +else + # redirect + print_status 302 + printf "Location: %s?%s\n" "$path" "${QUERY_STRING}" +fi + +echot/t5601-clone.sh +0
-0
t/t5601-clone.sh test_expect_success 'partial clone using HTTP' ' · line 763
Quoted 8 lines
t/t5601-clone.sh +5
-0@@ -756,6 +756,15 @@ test_expect_success 'partial clone using HTTP' ' partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server" ' +test_expect_success 'partial clone using HTTP with redirect' ' + _NONCE=`gen_nonce` && export _NONCE && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null &&
These lines are not indented with HT?
Don't redirect test output to /dev/null, which is done by test_expect_success for us. >/dev/null makes it less useful to run the test under "-v" option.
Quoted 6 lines
t/t5601-clone.sh +4
-0+ partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" +' + + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled.
Re: [PATCH] remote-curl: add testing for intelligent retry for HTTP
Quoted 10 lines
t/lib-httpd.sh check_access_log() { · line 316
Quoted 4 lines · Sean McAllister
t/lib-httpd.sh +4
-0+# generate a random 12 digit string +gen_nonce() { + test_copy_bytes 12 < /dev/urandom | tr -dc A-Za-z0-9 +}What is the randomness requirement of this application? IOW, what breaks if we just change this to "echo
0123456789ab"?
Quoted 3 lines
Or "date | git hash-object --stdin" for that matter?
We'd want to make our tests more predictiable, not less.
The randomness requirement is just that I need nonces to be unique during a single run of the HTTP server
as they uniquefy the files I put on disk to make the HTTP hack-ily stateful. I'd be fine with your date/hash-object
solution, but I don't know that it will help make the tests more predictable.
Quoted 39 lines
Quoted 26 lines
t/lib-httpd/error-ntime.sh new file +19
-0@@ -0,0 +1,79 @@ +#!/bin/sh + +# Script to simulate a transient error code with Retry-After header set. +# +# PATH_INFO must be of the form /<nonce>/<times>/<retcode>/<retry-after>/<path> +# (eg: /dc724af1/3/429/10/some/url) +# +# The <nonce> value uniquely identifies the URL, since we're simulating +# a stateful operation using a stateless protocol, we need a way to "namespace" +# URLs so that they don't step on each other. +# +# The first <times> times this endpoint is called, it will return the given +# <retcode>, and if the <retry-after> is non-negative, it will set the +# Retry-After head to that value. +# +# Subsequent calls will return a 302 redirect to <path>. +# +# Supported error codes are 429,502,503, and 504 +t/lib-httpd/error-ntime.sh line 29
Quoted 10 lines
t/lib-httpd/error-ntime.sh new file +10
-0+print_status() { + if [ "$1" -eq "302" ]; then printf "Status: 302 Found\n" + elif [ "$1" -eq "429" ]; then printf "Status: 429 Too Many Requests\n" + elif [ "$1" -eq "502" ]; then printf "Status: 502 Bad Gateway\n" + elif [ "$1" -eq "503" ]; then printf "Status: 503 Service Unavailable\n" + elif [ "$1" -eq "504" ]; then printf "Status: 504 Gateway Timeout\n" + else + printf "Status: 500 Internal Server Error\n" + fi + printf "Content-Type: text/plain\n"Style????? (I won't repeat this comment for the rest of this script)
Quoted 4 lines
I briefly wondered "oh, are t/lib-httpd/* scripts excempt from the coding guidelines?" but a quick look at them tells me that that is not the case.
I mistakenly thought the Makefile in t/ was linting these as well. I've gone back through and fixed formatting issues and removed non-posix constructs.
Quoted 69 lines
t/lib-httpd/error-ntime.sh line 40
Quoted 11 lines
t/lib-httpd/error-ntime.sh new file +11
-0+} + +# read in path split into cmoponents +IFS='/' +tokens=($PATH_INFO) + +# break out code/retry parts of path +nonce=${tokens[1]} +times=${tokens[2]} +code=${tokens[3]} +retry=${tokens[4]}You said
/bin/shupfront. Don't use non-POSIX shell arrays.Quoted 44 lines
t/lib-httpd/error-ntime.sh new file +39
-0+ +# get redirect path +cnt=0 +path="" +for ((ii=0; ii < ${#PATH_INFO}; ii++)); do + if [ "${PATH_INFO:${ii}:1}" == "/" ]; then + let cnt=${cnt}+1 + fi + if [ "${cnt}" -eq 5 ]; then + path="${PATH_INFO:${ii}}" + break + fi +done + +# leave a cookie for this request/retry count +state_file="request_${REMOTE_ADDR}_${nonce}_${times}_${code}_${retry}" + +if [ ! -f "$state_file" ]; then + echo 0 > "$state_file" +fi + + +read cnt < "$state_file" +if [ "$cnt" -lt "$times" ]; then + let cnt=cnt+1 + echo "$cnt" > "$state_file" + + # return error + print_status "$code" + if [ "$retry" -ge "0" ]; then + printf "Retry-After: %s\n" "$retry" + fi +else + # redirect + print_status 302 + printf "Location: %s?%s\n" "$path" "${QUERY_STRING}" +fi + +echot/t5601-clone.sh +0
-0t/t5601-clone.sh test_expect_success 'partial clone using HTTP' ' · line 763
Quoted 8 lines
t/t5601-clone.sh +5
-0@@ -756,6 +756,15 @@ test_expect_success 'partial clone using HTTP' ' partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server" ' +test_expect_success 'partial clone using HTTP with redirect' ' + _NONCE=`gen_nonce` && export _NONCE && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null && + curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" > /dev/null &&These lines are not indented with HT?
Quoted 3 lines
Don't redirect test output to
/dev/null, which is done bytest_expect_successfor us. >/dev/null makes it less useful to run the test under "-v" option.
Fixed in v2.
Quoted 6 lines
Quoted 6 lines
t/t5601-clone.sh +4
-0+ partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" +' + + # DO NOT add non-httpd-specific tests here, because the last part of this # test script is only executed when httpd is available and enabled.
Re: [PATCH] remote-curl: add testing for intelligent retry for HTTP
Quoted 12 lines · Sean McAllister
Quoted 12 lines
t/lib-httpd.sh check_access_log() { · line 316
Quoted 4 lines · Sean McAllister
t/lib-httpd.sh +4
-0+# generate a random 12 digit string +gen_nonce() { + test_copy_bytes 12 < /dev/urandom | tr -dc A-Za-z0-9 +}What is the randomness requirement of this application? IOW, what breaks if we just change this to "echo
0123456789ab"?Or "date | git hash-object --stdin" for that matter?
Quoted 7 lines
Quoted 1 line
We'd want to make our tests more predictiable, not less.
The randomness requirement is just that I need nonces to be unique during a single run of the HTTP server
as they uniquefy the files I put on disk to make the HTTP hack-ily stateful. I'd be fine with your date/hash-object
solution, but I don't know that it will help make the tests more predictable.
If so, would something like this be
global_counter_for_nonce=0
gen_nonce () {
global_counter_for_nonce=$(( global_counter_for_nonce + 1 )) &&
echo "$global_counter_for_nonce"
}more appropriate? It is utterly predictable and yields the same answer only once during a single run.
Re: [PATCH] remote-curl: add testing for intelligent retry for HTTP
Hi Junio,
Quoted 24 lines · Junio C Hamano
Quoted 12 lines · Sean McAllister
Quoted 12 lines
t/lib-httpd.sh check_access_log() { · line 316
Quoted 4 lines · Sean McAllister
t/lib-httpd.sh +4
-0+# generate a random 12 digit string +gen_nonce() { + test_copy_bytes 12 < /dev/urandom | tr -dc A-Za-z0-9 +}What is the randomness requirement of this application? IOW, what breaks if we just change this to "echo
0123456789ab"?Or "date | git hash-object --stdin" for that matter?
Quoted 7 lines
Quoted 1 line
We'd want to make our tests more predictiable, not less.
The randomness requirement is just that I need nonces to be unique during a single run of the HTTP server
as they uniquefy the files I put on disk to make the HTTP hack-ily stateful. I'd be fine with your date/hash-object
solution, but I don't know that it will help make the tests more predictable.If so, would something like this be
Quoted 8 lines
global_counter_for_nonce=0 gen_nonce () { global_counter_for_nonce=$(( global_counter_for_nonce + 1 )) && echo "$global_counter_for_nonce" }more appropriate? It is utterly predictable and yields the same answer only once during a single run.
We should also consider using test-tool genrandom <seed> instead (where <seed> would have to be predictable, but probably would have to change between gen_nonce() calls).
Ciao,
Dscho
Re: [PATCH] remote-curl: add testing for intelligent retry for HTTP
Quoted 3 lines · Johannes Schindelin
We should also consider using
test-tool genrandom <seed>instead (where<seed>would have to be predictable, but probably would have to change betweengen_nonce()calls).
Yup, that is exactly why I asked Sean about randomness requirement.
It turns out that they care only about uniqueness, so the comparison is between keeping an ever-incrementing counter and (1) echoing its current contents and/or (2) feeding it to "test-tool genrandom" as the seed. The complexity of the code _we_ need to write anew is the same, but echo would probably be a win in both the number of forks and cycles departments.
Thanks.