rsa: add pss encoding support

index 1815f55..bff1dc2 100644
@@ -2440,6 +2440,8 @@ static void usage(void)
 	printf(
 		"\n"
 		"  -a, --hashalgo     sha1 (default), sha224, sha256, sha384, sha512, streebog256, streebog512\n"
+		"  -e, --encoding     pkcs1 (default), pss\n"
+		"  -l, --saltlen      pss salt lenght, digest length is used by default\n"
 		"  -s, --imasig       make IMA signature\n"
 		"  -d, --imahash      make IMA hash\n"
 		"  -f, --sigfile      store IMA signature in .sig file instead of xattr\n"

@@ -2500,6 +2502,8 @@ static struct option opts[] = {
 	{"imasig", 0, 0, 's'},
 	{"imahash", 0, 0, 'd'},
 	{"hashalgo", 1, 0, 'a'},
+	{"encoding", 1, 0, 'e'},
+	{"saltlen", 1, 0, 'l'},
 	{"pass", 2, 0, 'p'},
 	{"sigfile", 0, 0, 'f'},
 	{"uuid", 2, 0, 'u'},

@@ -2567,7 +2571,7 @@ static char *get_password(void)
 
 int main(int argc, char *argv[])
 {
-	int err = 0, c, lind;
+	int err = 0, c, lind, val;
 	ENGINE *eng = NULL;
 
 #if !(OPENSSL_VERSION_NUMBER < 0x10100000)

@@ -2581,7 +2585,7 @@ int main(int argc, char *argv[])
 	g_argc = argc;
 
 	while (1) {
-		c = getopt_long(argc, argv, "hvnsda:op::fu::k:t:ri", opts, &lind);
+		c = getopt_long(argc, argv, "hvnsda:e:l:op::fu::k:t:ri", opts, &lind);
 		if (c == -1)
 			break;
 

@@ -2607,6 +2611,17 @@ int main(int argc, char *argv[])
 		case 'a':
 			imaevm_params.hash_algo = optarg;
 			break;
+		case 'e':
+			imaevm_params.encoding = optarg;
+			break;
+		case 'l':
+			val = atoi(optarg);
+			if (val <= 0) {
+				log_err("invalid pss salt len\n");
+				exit(1);
+			}
+			imaevm_params.saltlen = val;
+			break;
 		case 'p':
 			if (optarg)
 				imaevm_params.keypass = optarg;

index 4503919..4e2dc3a 100644
@@ -194,6 +194,8 @@ struct libimaevm_params {
 	int verbose;
 	int x509;
 	const char *hash_algo;
+	const char *encoding;
+	uint32_t saltlen;
 	const char *keyfile;
 	const char *keypass;
 };

index fa6c278..1bf6c67 100644
@@ -530,6 +530,13 @@ static int verify_hash_v2(const char *file, const unsigned char *hash, int size,
 	st = "EVP_PKEY_CTX_set_signature_md";
 	if (!EVP_PKEY_CTX_set_signature_md(ctx, md))
 		goto err;
+
+	if (imaevm_params.encoding &&
+	    !strcmp(imaevm_params.encoding, "pss")) {
+		if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING) <= 0)
+			goto err;
+	}
+
 	st = "EVP_PKEY_verify";
 	ret = EVP_PKEY_verify(ctx, sig + sizeof(*hdr),
 			      siglen - sizeof(*hdr), hash, size);

@@ -895,6 +902,7 @@ static int sign_hash_v2(const char *algo, const unsigned char *hash,
 	size_t sigsize;
 	const char *st;
 	uint32_t keyid;
+	int saltlen;
 
 	if (!hash) {
 		log_err("sign_hash_v2: hash is null\n");

@@ -947,6 +955,20 @@ static int sign_hash_v2(const char *algo, const unsigned char *hash,
 	st = "EVP_PKEY_CTX_set_signature_md";
 	if (!EVP_PKEY_CTX_set_signature_md(ctx, md))
 		goto err;
+
+	if (imaevm_params.encoding &&
+	    !strcmp(imaevm_params.encoding, "pss")) {
+		if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PSS_PADDING) <= 0)
+			goto err;
+
+		if (imaevm_params.saltlen)
+			saltlen = imaevm_params.saltlen;
+		else
+			saltlen = -1;
+		if (EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen) <= 0)
+			goto err;
+	}
+
 	st = "EVP_PKEY_sign";
 	sigsize = MAX_SIGNATURE_SIZE - sizeof(struct signature_v2_hdr) - 1;
 	if (!EVP_PKEY_sign(ctx, hdr->sig, &sigsize, hash, size))