crypto: aesni - Use zero-copy for gcm(aes) buffers that are partially contiguous

index 3bf3dcf29825..aef6c82b9ca7 100644
@@ -744,136 +744,148 @@ static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
 	return 0;
 }
 
+static bool is_mappable(struct scatterlist *sgl, unsigned long len)
+{
+	return sgl->length > 0 && len <= sgl->length &&
+	       (!PageHighMem(sg_page(sgl)) || sgl->offset + len <= PAGE_SIZE);
+}
+
+/*
+ * Maps the sglist buffer and returns a pointer to the mapped buffer in
+ * data_buf.
+ *
+ * If direct mapping is not feasible, then allocates a bounce buffer if one
+ * isn't already available in bounce_buf, and returns a pointer to the bounce
+ * buffer in data_buf.
+ *
+ * When the buffer is no longer needed, put_request_buffer() should be called on
+ * the data_buf and the bounce_buf should be freed using kfree().
+ */
+static int get_request_buffer(struct scatterlist *sgl,
+			      struct scatter_walk *sg_walk,
+			      unsigned long bounce_buf_size,
+			      u8 **data_buf, u8 **bounce_buf, bool *mapped)
+{
+	if (sg_is_last(sgl) && is_mappable(sgl, sgl->length)) {
+		*mapped = true;
+		scatterwalk_start(sg_walk, sgl);
+		*data_buf = scatterwalk_map(sg_walk);
+		return 0;
+	}
+
+	*mapped = false;
+
+	if (*bounce_buf == NULL) {
+		*bounce_buf = kmalloc(bounce_buf_size, GFP_ATOMIC);
+		if (unlikely(*bounce_buf == NULL))
+			return -ENOMEM;
+	}
+
+	*data_buf = *bounce_buf;
+	return 0;
+}
+
+static void put_request_buffer(u8 *data_buf, unsigned long len, bool mapped,
+			       struct scatter_walk *sg_walk, bool output)
+{
+	if (mapped) {
+		scatterwalk_unmap(data_buf);
+		scatterwalk_advance(sg_walk, len);
+		scatterwalk_done(sg_walk, output, 0);
+	}
+}
+
+/*
+ * Performs the encryption/decryption operation for the given request. The src
+ * and dst sglists in the request are directly mapped if possible. Otherwise, a
+ * bounce buffer is allocated and used to copy the data from the src or to the
+ * dst, or both.
+ */
+static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
+			u8 *hash_subkey, u8 *iv, void *aes_ctx, bool decrypt)
+{
+	u8 *src, *dst, *assoc, *bounce_buf = NULL;
+	bool src_mapped = false, dst_mapped = false;
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
+	unsigned long data_len = req->cryptlen - (decrypt ? auth_tag_len : 0);
+	struct scatter_walk src_sg_walk;
+	struct scatter_walk dst_sg_walk = {};
+	int retval = 0;
+	unsigned long bounce_buf_size = data_len + auth_tag_len + req->assoclen;
+
+	if (auth_tag_len > 16)
+		return -EINVAL;
+
+	retval = get_request_buffer(req->src, &src_sg_walk, bounce_buf_size,
+				    &assoc, &bounce_buf, &src_mapped);
+	if (retval)
+		goto exit;
+
+	src = assoc + req->assoclen;
+
+	if (req->src == req->dst) {
+		dst = src;
+		dst_mapped = src_mapped;
+	} else {
+		retval = get_request_buffer(req->dst, &dst_sg_walk,
+					    bounce_buf_size, &dst, &bounce_buf,
+					    &dst_mapped);
+		if (retval)
+			goto exit;
+
+		dst += req->assoclen;
+	}
+
+	if (!src_mapped)
+		scatterwalk_map_and_copy(bounce_buf, req->src, 0,
+					 req->assoclen + req->cryptlen, 0);
+
+	kernel_fpu_begin();
+
+	if (decrypt) {
+		u8 gen_auth_tag[16];
+
+		aesni_gcm_dec_tfm(aes_ctx, dst, src, data_len, iv,
+				  hash_subkey, assoc, assoclen,
+				  gen_auth_tag, auth_tag_len);
+		/* Compare generated tag with passed in tag. */
+		if (crypto_memneq(src + data_len, gen_auth_tag, auth_tag_len))
+			retval = -EBADMSG;
+
+	} else
+		aesni_gcm_enc_tfm(aes_ctx, dst, src, data_len, iv,
+				  hash_subkey, assoc, assoclen,
+				  dst + data_len, auth_tag_len);
+
+	kernel_fpu_end();
+
+	if (!dst_mapped)
+		scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
+					 data_len + (decrypt ? 0 : auth_tag_len),
+					 1);
+exit:
+	if (req->dst != req->src)
+		put_request_buffer(dst - req->assoclen, req->dst->length,
+				   dst_mapped, &dst_sg_walk, true);
+
+	put_request_buffer(assoc, req->src->length, src_mapped, &src_sg_walk,
+			   false);
+
+	kfree(bounce_buf);
+	return retval;
+}
+
 static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
 {
-	u8 one_entry_in_sg = 0;
-	u8 *src, *dst, *assoc;
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
-	struct scatter_walk src_sg_walk;
-	struct scatter_walk dst_sg_walk = {};
-
-	if (sg_is_last(req->src) &&
-	    (!PageHighMem(sg_page(req->src)) ||
-	    req->src->offset + req->src->length <= PAGE_SIZE) &&
-	    sg_is_last(req->dst) &&
-	    (!PageHighMem(sg_page(req->dst)) ||
-	    req->dst->offset + req->dst->length <= PAGE_SIZE)) {
-		one_entry_in_sg = 1;
-		scatterwalk_start(&src_sg_walk, req->src);
-		assoc = scatterwalk_map(&src_sg_walk);
-		src = assoc + req->assoclen;
-		dst = src;
-		if (unlikely(req->src != req->dst)) {
-			scatterwalk_start(&dst_sg_walk, req->dst);
-			dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
-		}
-	} else {
-		/* Allocate memory for src, dst, assoc */
-		assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
-			GFP_ATOMIC);
-		if (unlikely(!assoc))
-			return -ENOMEM;
-		scatterwalk_map_and_copy(assoc, req->src, 0,
-					 req->assoclen + req->cryptlen, 0);
-		src = assoc + req->assoclen;
-		dst = src;
-	}
-
-	kernel_fpu_begin();
-	aesni_gcm_enc_tfm(aes_ctx, dst, src, req->cryptlen, iv,
-			  hash_subkey, assoc, assoclen,
-			  dst + req->cryptlen, auth_tag_len);
-	kernel_fpu_end();
-
-	/* The authTag (aka the Integrity Check Value) needs to be written
-	 * back to the packet. */
-	if (one_entry_in_sg) {
-		if (unlikely(req->src != req->dst)) {
-			scatterwalk_unmap(dst - req->assoclen);
-			scatterwalk_advance(&dst_sg_walk, req->dst->length);
-			scatterwalk_done(&dst_sg_walk, 1, 0);
-		}
-		scatterwalk_unmap(assoc);
-		scatterwalk_advance(&src_sg_walk, req->src->length);
-		scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
-	} else {
-		scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
-					 req->cryptlen + auth_tag_len, 1);
-		kfree(assoc);
-	}
-	return 0;
+	return gcmaes_crypt(req, assoclen, hash_subkey, iv, aes_ctx, false);
 }
 
 static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 			  u8 *hash_subkey, u8 *iv, void *aes_ctx)
 {
-	u8 one_entry_in_sg = 0;
-	u8 *src, *dst, *assoc;
-	unsigned long tempCipherLen = 0;
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
-	u8 authTag[16];
-	struct scatter_walk src_sg_walk;
-	struct scatter_walk dst_sg_walk = {};
-	int retval = 0;
-
-	tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
-
-	if (sg_is_last(req->src) &&
-	    (!PageHighMem(sg_page(req->src)) ||
-	    req->src->offset + req->src->length <= PAGE_SIZE) &&
-	    sg_is_last(req->dst) &&
-	    (!PageHighMem(sg_page(req->dst)) ||
-	    req->dst->offset + req->dst->length <= PAGE_SIZE)) {
-		one_entry_in_sg = 1;
-		scatterwalk_start(&src_sg_walk, req->src);
-		assoc = scatterwalk_map(&src_sg_walk);
-		src = assoc + req->assoclen;
-		dst = src;
-		if (unlikely(req->src != req->dst)) {
-			scatterwalk_start(&dst_sg_walk, req->dst);
-			dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
-		}
-	} else {
-		/* Allocate memory for src, dst, assoc */
-		assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
-		if (!assoc)
-			return -ENOMEM;
-		scatterwalk_map_and_copy(assoc, req->src, 0,
-					 req->assoclen + req->cryptlen, 0);
-		src = assoc + req->assoclen;
-		dst = src;
-	}
-
-
-	kernel_fpu_begin();
-	aesni_gcm_dec_tfm(aes_ctx, dst, src, tempCipherLen, iv,
-			  hash_subkey, assoc, assoclen,
-			  authTag, auth_tag_len);
-	kernel_fpu_end();
-
-	/* Compare generated tag with passed in tag. */
-	retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
-		-EBADMSG : 0;
-
-	if (one_entry_in_sg) {
-		if (unlikely(req->src != req->dst)) {
-			scatterwalk_unmap(dst - req->assoclen);
-			scatterwalk_advance(&dst_sg_walk, req->dst->length);
-			scatterwalk_done(&dst_sg_walk, 1, 0);
-		}
-		scatterwalk_unmap(assoc);
-		scatterwalk_advance(&src_sg_walk, req->src->length);
-		scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
-	} else {
-		scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
-					 tempCipherLen, 1);
-		kfree(assoc);
-	}
-	return retval;
-
+	return gcmaes_crypt(req, assoclen, hash_subkey, iv, aes_ctx, true);
 }
 
 static int helper_rfc4106_encrypt(struct aead_request *req)

index aef6c82b9ca7..8615ecb0ff1e 100644
@@ -750,6 +750,11 @@ static bool is_mappable(struct scatterlist *sgl, unsigned long len)
 	       (!PageHighMem(sg_page(sgl)) || sgl->offset + len <= PAGE_SIZE);
 }
 
+static u8 *map_buffer(struct scatterlist *sgl)
+{
+	return kmap_atomic(sg_page(sgl)) + sgl->offset;
+}
+
 /*
  * Maps the sglist buffer and returns a pointer to the mapped buffer in
  * data_buf.

@@ -762,14 +767,12 @@ static bool is_mappable(struct scatterlist *sgl, unsigned long len)
  * the data_buf and the bounce_buf should be freed using kfree().
  */
 static int get_request_buffer(struct scatterlist *sgl,
-			      struct scatter_walk *sg_walk,
 			      unsigned long bounce_buf_size,
 			      u8 **data_buf, u8 **bounce_buf, bool *mapped)
 {
 	if (sg_is_last(sgl) && is_mappable(sgl, sgl->length)) {
 		*mapped = true;
-		scatterwalk_start(sg_walk, sgl);
-		*data_buf = scatterwalk_map(sg_walk);
+		*data_buf = map_buffer(sgl);
 		return 0;
 	}
 

@@ -785,14 +788,10 @@ static int get_request_buffer(struct scatterlist *sgl,
 	return 0;
 }
 
-static void put_request_buffer(u8 *data_buf, unsigned long len, bool mapped,
-			       struct scatter_walk *sg_walk, bool output)
+static void put_request_buffer(u8 *data_buf, bool mapped)
 {
-	if (mapped) {
-		scatterwalk_unmap(data_buf);
-		scatterwalk_advance(sg_walk, len);
-		scatterwalk_done(sg_walk, output, 0);
-	}
+	if (mapped)
+		kunmap_atomic(data_buf);
 }
 
 /*

@@ -809,16 +808,14 @@ static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 	unsigned long data_len = req->cryptlen - (decrypt ? auth_tag_len : 0);
-	struct scatter_walk src_sg_walk;
-	struct scatter_walk dst_sg_walk = {};
 	int retval = 0;
 	unsigned long bounce_buf_size = data_len + auth_tag_len + req->assoclen;
 
 	if (auth_tag_len > 16)
 		return -EINVAL;
 
-	retval = get_request_buffer(req->src, &src_sg_walk, bounce_buf_size,
-				    &assoc, &bounce_buf, &src_mapped);
+	retval = get_request_buffer(req->src, bounce_buf_size, &assoc,
+				    &bounce_buf, &src_mapped);
 	if (retval)
 		goto exit;
 

@@ -828,9 +825,8 @@ static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 		dst = src;
 		dst_mapped = src_mapped;
 	} else {
-		retval = get_request_buffer(req->dst, &dst_sg_walk,
-					    bounce_buf_size, &dst, &bounce_buf,
-					    &dst_mapped);
+		retval = get_request_buffer(req->dst, bounce_buf_size, &dst,
+					    &bounce_buf, &dst_mapped);
 		if (retval)
 			goto exit;
 

@@ -866,11 +862,9 @@ static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 					 1);
 exit:
 	if (req->dst != req->src)
-		put_request_buffer(dst - req->assoclen, req->dst->length,
-				   dst_mapped, &dst_sg_walk, true);
+		put_request_buffer(dst - req->assoclen, dst_mapped);
 
-	put_request_buffer(assoc, req->src->length, src_mapped, &src_sg_walk,
-			   false);
+	put_request_buffer(assoc, src_mapped);
 
 	kfree(bounce_buf);
 	return retval;

index 8615ecb0ff1e..b9a7d9ea5ba7 100644
@@ -756,42 +756,91 @@ static u8 *map_buffer(struct scatterlist *sgl)
 }
 
 /*
- * Maps the sglist buffer and returns a pointer to the mapped buffer in
- * data_buf.
+ * Maps the sglist buffer and returns pointers to the mapped buffers in assoc,
+ * data and (optionally) auth_tag.
  *
  * If direct mapping is not feasible, then allocates a bounce buffer if one
- * isn't already available in bounce_buf, and returns a pointer to the bounce
- * buffer in data_buf.
+ * isn't already available in bounce_buf, and returns pointers within the bounce
+ * buffer in assoc, data and auth_tag.
  *
- * When the buffer is no longer needed, put_request_buffer() should be called on
- * the data_buf and the bounce_buf should be freed using kfree().
+ * When the buffers are no longer needed, put_request_buffers() should be called
+ * and the bounce_buf should be freed using kfree().
  */
-static int get_request_buffer(struct scatterlist *sgl,
-			      unsigned long bounce_buf_size,
-			      u8 **data_buf, u8 **bounce_buf, bool *mapped)
+static int get_request_buffers(struct scatterlist *sgl,
+			       unsigned long assoc_len, unsigned long data_len,
+			       unsigned long auth_tag_len,
+			       u8 **assoc, u8 **data, u8 **auth_tag,
+			       u8 **bounce_buf, bool *mapped)
 {
-	if (sg_is_last(sgl) && is_mappable(sgl, sgl->length)) {
+	struct scatterlist sgl_data_chain[2], sgl_auth_tag_chain[2];
+	struct scatterlist *sgl_data, *sgl_auth_tag;
+
+	sgl_data = scatterwalk_ffwd(sgl_data_chain, sgl, assoc_len);
+	sgl_auth_tag = scatterwalk_ffwd(sgl_auth_tag_chain, sgl,
+					assoc_len + data_len);
+
+	if (is_mappable(sgl, assoc_len) && is_mappable(sgl_data, data_len) &&
+	    (auth_tag == NULL || is_mappable(sgl_auth_tag, auth_tag_len))) {
 		*mapped = true;
-		*data_buf = map_buffer(sgl);
+
+		*assoc = map_buffer(sgl);
+
+		if (sgl->length >= assoc_len + data_len)
+			*data = *assoc + assoc_len;
+		else
+			*data = map_buffer(sgl_data);
+
+		if (auth_tag != NULL) {
+			if (sgl_data->length >= data_len + auth_tag_len)
+				*auth_tag = *data + data_len;
+			else
+				*auth_tag = map_buffer(sgl_auth_tag);
+		}
+
 		return 0;
 	}
 
 	*mapped = false;
 
 	if (*bounce_buf == NULL) {
-		*bounce_buf = kmalloc(bounce_buf_size, GFP_ATOMIC);
+		*bounce_buf = kmalloc(assoc_len + data_len + auth_tag_len,
+				      GFP_ATOMIC);
 		if (unlikely(*bounce_buf == NULL))
 			return -ENOMEM;
 	}
 
-	*data_buf = *bounce_buf;
+	*assoc = *bounce_buf;
+	*data = *assoc + assoc_len;
+
+	if (auth_tag != NULL)
+		*auth_tag = *data + data_len;
+
 	return 0;
 }
 
-static void put_request_buffer(u8 *data_buf, bool mapped)
+static void put_request_buffers(struct scatterlist *sgl, bool mapped,
+				u8 *assoc, u8 *data, u8 *auth_tag,
+				unsigned long assoc_len,
+				unsigned long data_len,
+				unsigned long auth_tag_len)
 {
-	if (mapped)
-		kunmap_atomic(data_buf);
+	struct scatterlist sgl_data_chain[2];
+	struct scatterlist *sgl_data;
+
+	if (!mapped)
+		return;
+
+	sgl_data = scatterwalk_ffwd(sgl_data_chain, sgl, assoc_len);
+
+	/* The unmaps need to be done in reverse order of the maps. */
+
+	if (auth_tag != NULL && sgl_data->length < data_len + auth_tag_len)
+		kunmap_atomic(auth_tag);
+
+	if (sgl->length < assoc_len + data_len)
+		kunmap_atomic(data);
+
+	kunmap_atomic(assoc);
 }
 
 /*

@@ -803,34 +852,38 @@ static void put_request_buffer(u8 *data_buf, bool mapped)
 static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 			u8 *hash_subkey, u8 *iv, void *aes_ctx, bool decrypt)
 {
-	u8 *src, *dst, *assoc, *bounce_buf = NULL;
+	u8 *src, *src_assoc;
+	u8 *dst, *dst_assoc;
+	u8 *auth_tag;
+	u8 *bounce_buf = NULL;
 	bool src_mapped = false, dst_mapped = false;
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	unsigned long auth_tag_len = crypto_aead_authsize(tfm);
 	unsigned long data_len = req->cryptlen - (decrypt ? auth_tag_len : 0);
 	int retval = 0;
-	unsigned long bounce_buf_size = data_len + auth_tag_len + req->assoclen;
 
 	if (auth_tag_len > 16)
 		return -EINVAL;
 
-	retval = get_request_buffer(req->src, bounce_buf_size, &assoc,
-				    &bounce_buf, &src_mapped);
+	retval = get_request_buffers(req->src, req->assoclen, data_len,
+				     auth_tag_len, &src_assoc, &src,
+				     (decrypt || req->src == req->dst)
+				     ? &auth_tag : NULL,
+				     &bounce_buf, &src_mapped);
 	if (retval)
 		goto exit;
 
-	src = assoc + req->assoclen;
-
 	if (req->src == req->dst) {
+		dst_assoc = src_assoc;
 		dst = src;
 		dst_mapped = src_mapped;
 	} else {
-		retval = get_request_buffer(req->dst, bounce_buf_size, &dst,
-					    &bounce_buf, &dst_mapped);
+		retval = get_request_buffers(req->dst, req->assoclen, data_len,
+					     auth_tag_len, &dst_assoc, &dst,
+					     decrypt ? NULL : &auth_tag,
+					     &bounce_buf, &dst_mapped);
 		if (retval)
 			goto exit;
-
-		dst += req->assoclen;
 	}
 
 	if (!src_mapped)

@@ -843,16 +896,16 @@ static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 		u8 gen_auth_tag[16];
 
 		aesni_gcm_dec_tfm(aes_ctx, dst, src, data_len, iv,
-				  hash_subkey, assoc, assoclen,
+				  hash_subkey, src_assoc, assoclen,
 				  gen_auth_tag, auth_tag_len);
 		/* Compare generated tag with passed in tag. */
-		if (crypto_memneq(src + data_len, gen_auth_tag, auth_tag_len))
+		if (crypto_memneq(auth_tag, gen_auth_tag, auth_tag_len))
 			retval = -EBADMSG;
 
 	} else
 		aesni_gcm_enc_tfm(aes_ctx, dst, src, data_len, iv,
-				  hash_subkey, assoc, assoclen,
-				  dst + data_len, auth_tag_len);
+				  hash_subkey, src_assoc, assoclen,
+				  auth_tag, auth_tag_len);
 
 	kernel_fpu_end();
 

@@ -862,9 +915,13 @@ static int gcmaes_crypt(struct aead_request *req, unsigned int assoclen,
 					 1);
 exit:
 	if (req->dst != req->src)
-		put_request_buffer(dst - req->assoclen, dst_mapped);
+		put_request_buffers(req->dst, dst_mapped, dst_assoc, dst,
+				    decrypt ? NULL : auth_tag,
+				    req->assoclen, data_len, auth_tag_len);
 
-	put_request_buffer(assoc, src_mapped);
+	put_request_buffers(req->src, src_mapped, src_assoc, src,
+			    (decrypt || req->src == req->dst) ? auth_tag : NULL,
+			    req->assoclen, data_len, auth_tag_len);
 
 	kfree(bounce_buf);
 	return retval;