Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Aug 17, 2018

lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit 35e2d88fad2151f272af60babb5e6c98922d0bcd:

Fix compilation on centos 7 (2018-08-15 12:06:14 -0600)

are available in the git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 9ee669faa39003c2317e5df892314bcfcee069e3:

configure: avoid pkg-config usage for http engine (2018-08-16 18:58:27 +0200)

----------------------------------------------------------------
David Disseldorp (3):
configure: use pkg-config to detect libcurl & openssl engines/http: support openssl < 1.1.0
configure: avoid pkg-config usage for http engine

Jens Axboe (3):
Merge branch 'http_older_openssl' of https://github.com/ddiss/fio
engines/http: fix use of uninitialized variable
Merge branch 'wip-http-swift' of https://github.com/l-mb/fio

Lars Marowsky-Bree (1):
engines/http: Add support for Swift storage backend

 HOWTO                    |  17 ++++--
 configure                |  35 ++++++++++--
 engines/http.c           | 135 +++++++++++++++++++++++++++++++++++++++++------
 examples/http-s3.fio     |   4 +-
 examples/http-swift.fio  |  32 +++++++++++
 examples/http-webdav.fio |   4 +-
 fio.1                    |  16 ++++--
 7 files changed, 209 insertions(+), 34 deletions(-)
 create mode 100644 examples/http-swift.fio

---

Diff of recent changes:

HOWTO +12 -5
index c77dad1..743144f 100644
@@ -2137,14 +2137,16 @@ with the caveat that when used on the command line, they must come after the
 
 	Password for HTTP authentication.
 
-.. option:: https=bool : [http]
+.. option:: https=str : [http]
 
-	Enable HTTPS instead of http. Default is **0**
+	Enable HTTPS instead of http. *on* enables HTTPS; *insecure*
+	will enable HTTPS, but disable SSL peer verification (use with
+	caution!). Default is **off**
 
-.. option:: http_s3=bool : [http]
+.. option:: http_mode=str : [http]
 
-	Enable S3 specific HTTP headers such as authenticating requests
-	with AWS Signature Version 4. Default is **0**
+	Which HTTP access mode to use: *webdav*, *swift*, or *s3*.
+	Default is **webdav**
 
 .. option:: http_s3_region=str : [http]
 
@@ -2159,6 +2161,11 @@ with the caveat that when used on the command line, they must come after the
 
 	The S3 key/access id.
 
+.. option:: http_swift_auth_token=str : [http]
+
+	The Swift auth token. See the example configuration file on how
+	to retrieve this.
+
 .. option:: http_verbose=int : [http]
 
 	Enable verbose requests from libcurl. Useful for debugging. 1
configure +31 -4
index 103ea94..a03f7fa 100755
@@ -14,12 +14,13 @@ else
 fi
 
 TMPC="${TMPDIR1}/fio-conf-${RANDOM}-$$-${RANDOM}.c"
+TMPC2="${TMPDIR1}/fio-conf-${RANDOM}-$$-${RANDOM}-2.c"
 TMPO="${TMPDIR1}/fio-conf-${RANDOM}-$$-${RANDOM}.o"
 TMPE="${TMPDIR1}/fio-conf-${RANDOM}-$$-${RANDOM}.exe"
 
 # NB: do not call "exit" in the trap handler; this is buggy with some shells;
 # see <[email protected]>
-trap "rm -f $TMPC $TMPO $TMPE" EXIT INT QUIT TERM
+trap "rm -f $TMPC $TMPC2 $TMPO $TMPE" EXIT INT QUIT TERM
 
 rm -rf config.log
 
@@ -1573,6 +1574,7 @@ print_config "IPv6 helpers" "$ipv6"
 if test "$http" != "yes" ; then
   http="no"
 fi
+# check for openssl >= 1.1.0, which uses an opaque HMAC_CTX pointer
 cat > $TMPC << EOF
 #include <curl/curl.h>
 #include <openssl/hmac.h>
@@ -1591,9 +1593,34 @@ int main(int argc, char **argv)
   return 0;
 }
 EOF
-if test "$disable_http" != "yes"  && compile_prog "" "-lcurl -lssl -lcrypto" "curl"; then
-  LIBS="-lcurl -lssl -lcrypto $LIBS"
-  http="yes"
+# openssl < 1.1.0 uses the HMAC_CTX type directly
+cat > $TMPC2 << EOF
+#include <curl/curl.h>
+#include <openssl/hmac.h>
+
+int main(int argc, char **argv)
+{
+  CURL *curl;
+  HMAC_CTX ctx;
+
+  curl = curl_easy_init();
+  curl_easy_cleanup(curl);
+
+  HMAC_CTX_init(&ctx);
+  HMAC_CTX_cleanup(&ctx);
+  return 0;
+}
+EOF
+if test "$disable_http" != "yes"; then
+  HTTP_LIBS="-lcurl -lssl -lcrypto"
+  if compile_prog "" "$HTTP_LIBS" "curl-new-ssl"; then
+    output_sym "CONFIG_HAVE_OPAQUE_HMAC_CTX"
+    http="yes"
+    LIBS="$HTTP_LIBS $LIBS"
+  elif mv $TMPC2 $TMPC && compile_prog "" "$HTTP_LIBS" "curl-old-ssl"; then
+    http="yes"
+    LIBS="$HTTP_LIBS $LIBS"
+  fi
 fi
 print_config "http engine" "$http"
 
engines/http.c +119 -16

236 lines not on this page · raw message

examples/http-s3.fio +2 -2
index a9805da..2dcae36 100644
@@ -9,8 +9,8 @@ name=test
 direct=1
 filename=/larsmb-fio-test/object
 http_verbose=0
-https=1
-http_s3=1
+https=on
+http_mode=s3
 http_s3_key=${S3_KEY}
 http_s3_keyid=${S3_ID}
 http_host=s3.eu-central-1.amazonaws.com
examples/http-swift.fio new file +32 -0
new file mode 100644
index 0000000..b591adb
@@ -0,0 +1,32 @@
+[global]
+ioengine=http
+rw=randwrite
+name=test
+direct=1
+http_verbose=0
+http_mode=swift
+https=on
+# This is the hostname and port portion of the public access link for
+# the container:
+http_host=swift.srv.openstack.local:8081
+filename_format=/swift/v1/fio-test/bucket.$jobnum
+group_reporting
+bs=64k
+size=1M
+# Currently, fio cannot yet generate the Swift Auth-Token itself.
+# You need to set this prior to running fio via
+# eval $(openstack token issue -f shell --prefix SWIFT_) ; export SWIFT_id
+http_swift_auth_token=${SWIFT_id}
+
+[create]
+numjobs=1
+rw=randwrite
+io_size=256k
+verify=sha256
+
+# This will delete all created objects again
+[trim]
+stonewall
+numjobs=1
+rw=trim
+io_size=64k
examples/http-webdav.fio +2 -2
index c0624f8..2d1ca73 100644
@@ -4,8 +4,8 @@ rw=randwrite
 name=test
 direct=1
 http_verbose=0
-http_s3=0
-https=0
+http_mode=webdav
+https=off
 http_host=localhost
 filename_format=/dav/bucket.$jobnum
 group_reporting
fio.1 +11 -5
index 883a31b..73a0422 100644
@@ -1829,12 +1829,14 @@ Username for HTTP authentication.
 .BI (http)http_pass \fR=\fPstr
 Password for HTTP authentication.
 .TP
-.BI (http)https \fR=\fPbool
-Whether to use HTTPS instead of plain HTTP. Default is \fB0\fR.
+.BI (http)https \fR=\fPstr
+Whether to use HTTPS instead of plain HTTP. \fRon\fP enables HTTPS;
+\fRinsecure\fP will enable HTTPS, but disable SSL peer verification (use
+with caution!).  Default is \fBoff\fR.
 .TP
-.BI (http)http_s3 \fR=\fPbool
-Include S3 specific HTTP headers such as authenticating requests with
-AWS Signature Version 4. Default is \fB0\fR.
+.BI (http)http_mode \fR=\fPstr
+Which HTTP access mode to use: webdav, swift, or s3. Default is
+\fBwebdav\fR.
 .TP
 .BI (http)http_s3_region \fR=\fPstr
 The S3 region/zone to include in the request. Default is \fBus-east-1\fR.
@@ -1845,6 +1847,10 @@ The S3 secret key.
 .BI (http)http_s3_keyid \fR=\fPstr
 The S3 key/access id.
 .TP
+.BI (http)http_swift_auth_token \fR=\fPstr
+The Swift auth token. See the example configuration file on how to
+retrieve this.
+.TP
 .BI (http)http_verbose \fR=\fPint
 Enable verbose requests from libcurl. Useful for debugging. 1 turns on
 verbose logging from libcurl, 2 additionally enables HTTP IO tracing.