Drivers for Alibaba YCC (Yitian Cryptography Complex) cryptographic accelerator

148 lines not on this page · raw message

444 lines not on this page · raw message

117 lines not on this page · raw message

new file mode 100644
index 00000000..8318a6f
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __YCC_ISR_H
+
+#include "ycc_dev.h"
+
+int ycc_enable_msix(struct ycc_dev *ydev);
+void ycc_disable_msix(struct ycc_dev *ydev);
+int ycc_alloc_irqs(struct ycc_dev *ydev);
+void ycc_free_irqs(struct ycc_dev *ydev);
+int ycc_init_global_err(struct ycc_dev *ydev);
+void ycc_deinit_global_err(struct ycc_dev *ydev);
+#endif

index e5f76af..dda82b0 100644
@@ -104,10 +104,16 @@ struct ycc_dev {
 	struct ycc_bar ycc_bars[4];
 	struct ycc_dev *assoc_dev;
 
+	int max_desc;
+	int user_rings;
 	bool is_polling;
 	unsigned long status;
 	struct workqueue_struct *dev_err_q;
 	char err_irq_name[32];
+
+	struct ycc_ring *rings;
+	unsigned long ring_bitmap;
+
 	struct work_struct work;
 	char *msi_name[48];
 	struct dentry *debug_dir;

128 lines not on this page · raw message

index f2f751c..cd2a2d7 100644
@@ -38,6 +38,13 @@ static irqreturn_t ycc_g_err_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+/*
+ * TODO: will implement when ycc ring actually work.
+ */
+void ycc_resp_process(uintptr_t ring_addr)
+{
+}
+
 int ycc_enable_msix(struct ycc_dev *ydev)
 {
 	struct pci_dev *rcec_pdev = ydev->assoc_dev->pdev;

562 lines not on this page · raw message

111 lines not on this page · raw message

index cd2a2d7..a86c8d7 100644
@@ -12,6 +12,17 @@
 #include <linux/interrupt.h>
 
 #include "ycc_isr.h"
+#include "ycc_dev.h"
+#include "ycc_ring.h"
+
+
+static irqreturn_t ycc_resp_isr(int irq, void *data)
+{
+	struct ycc_ring *ring = (struct ycc_ring *)data;
+
+	tasklet_hi_schedule(&ring->resp_handler);
+	return IRQ_HANDLED;
+}
 
 /*
  * TODO: will implement when ycc ring actually work.

@@ -38,11 +49,13 @@ static irqreturn_t ycc_g_err_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-/*
- * TODO: will implement when ycc ring actually work.
- */
 void ycc_resp_process(uintptr_t ring_addr)
 {
+	struct ycc_ring *ring = (struct ycc_ring *)ring_addr;
+
+	ycc_dequeue(ring);
+	if (ring->ydev->is_polling)
+		tasklet_hi_schedule(&ring->resp_handler);
 }
 
 int ycc_enable_msix(struct ycc_dev *ydev)

@@ -83,34 +96,89 @@ static void ycc_cleanup_global_err_workqueue(struct ycc_dev *ydev)
 		destroy_workqueue(ydev->dev_err_q);
 }
 
-/*
- * TODO: Just request irq for global err. Irq for each ring
- * will be requested when ring actually work.
- */
 int ycc_alloc_irqs(struct ycc_dev *ydev)
 {
 	struct pci_dev *rcec_pdev = ydev->assoc_dev->pdev;
 	int num = ydev->is_vf ? 1 : YCC_RINGPAIR_NUM;
-	int ret;
+	int cpu, cpus = num_online_cpus();
+	int ret, i, j;
 
+	/* The 0 - (YCC_RINGPAIR_NUM-1) are rings irqs, the last one is dev error irq */
 	sprintf(ydev->err_irq_name, "ycc_dev_%d_global_err", ydev->id);
 	ret = request_irq(pci_irq_vector(rcec_pdev, num),
 			  ycc_g_err_isr, 0, ydev->err_irq_name, ydev);
-	if (ret)
+	if (ret) {
 		pr_err("Failed to alloc global irq interrupt for dev: %d\n", ydev->id);
+		goto out;
+	}
+
+	if (ydev->is_polling)
+		goto out;
+
+	for (i = 0; i < num; i++) {
+		if (ydev->rings[i].type != KERN_RING)
+			continue;
+
+		ydev->msi_name[i] = kzalloc(16, GFP_KERNEL);
+		if (!ydev->msi_name[i])
+			goto free_irq;
+		snprintf(ydev->msi_name[i], 16, "ycc_ring_%d", i);
+		ret = request_irq(pci_irq_vector(rcec_pdev, i), ycc_resp_isr,
+				  0, ydev->msi_name[i], &ydev->rings[i]);
+		if (ret) {
+			kfree(ydev->msi_name[i]);
+			goto free_irq;
+		}
+		if (!ydev->is_vf)
+			cpu = (i % YCC_RINGPAIR_NUM) % cpus;
+		else
+			cpu = smp_processor_id() % cpus;
+
+		ret = irq_set_affinity_hint(pci_irq_vector(rcec_pdev, i),
+					    get_cpu_mask(cpu));
+		if (ret) {
+			free_irq(pci_irq_vector(rcec_pdev, i), &ydev->rings[i]);
+			kfree(ydev->msi_name[i]);
+			goto free_irq;
+		}
+	}
+
+	return 0;
+
+free_irq:
+	for (j = 0; j < i; j++) {
+		if (ydev->rings[i].type != KERN_RING)
+			continue;
+
+		free_irq(pci_irq_vector(rcec_pdev, j), &ydev->rings[j]);
+		kfree(ydev->msi_name[j]);
+	}
+	free_irq(pci_irq_vector(rcec_pdev, num), ydev);
+out:
 
 	return ret;
 }
 
-/*
- * TODO: Same as the allocate action.
- */
 void ycc_free_irqs(struct ycc_dev *ydev)
 {
 	struct pci_dev *rcec_pdev = ydev->assoc_dev->pdev;
 	int num = ydev->is_vf ? 1 : YCC_RINGPAIR_NUM;
+	int i;
 
+	/* Free device err irq */
 	free_irq(pci_irq_vector(rcec_pdev, num), ydev);
+
+	if (ydev->is_polling)
+		return;
+
+	for (i = 0; i < num; i++) {
+		if (ydev->rings[i].type != KERN_RING)
+			continue;
+
+		irq_set_affinity_hint(pci_irq_vector(rcec_pdev, i), NULL);
+		free_irq(pci_irq_vector(rcec_pdev, i), &ydev->rings[i]);
+		kfree(ydev->msi_name[i]);
+	}
 }
 
 int ycc_init_global_err(struct ycc_dev *ydev)

index a86c8d7..28f8918 100644
@@ -15,6 +15,8 @@
 #include "ycc_dev.h"
 #include "ycc_ring.h"
 
+extern void ycc_clear_cmd_ring(struct ycc_ring *ring);
+extern void ycc_clear_resp_ring(struct ycc_ring *ring);
 
 static irqreturn_t ycc_resp_isr(int irq, void *data)
 {

@@ -24,11 +26,93 @@ static irqreturn_t ycc_resp_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-/*
- * TODO: will implement when ycc ring actually work.
- */
+static void ycc_fatal_error(struct ycc_dev *ydev)
+{
+	struct ycc_ring *ring;
+	int i;
+
+	for (i = 0; i < YCC_RINGPAIR_NUM; i++) {
+		ring = ydev->rings + i;
+
+		if (ring->type != KERN_RING)
+			continue;
+
+		spin_lock_bh(&ring->lock);
+		ycc_clear_cmd_ring(ring);
+		spin_unlock_bh(&ring->lock);
+
+		ycc_clear_resp_ring(ring);
+	}
+}
+
 static void ycc_process_global_err(struct work_struct *work)
 {
+	struct ycc_dev *ydev = container_of(work, struct ycc_dev, work);
+	struct ycc_bar *cfg_bar = &ydev->ycc_bars[YCC_SEC_CFG_BAR];
+	struct ycc_ring *ring;
+	u32 hclk_err, xclk_err;
+	u32 xclk_ecc_uncor_err_0, xclk_ecc_uncor_err_1;
+	u32 hclk_ecc_uncor_err;
+	int i;
+
+	if (pci_wait_for_pending_transaction(ydev->pdev))
+		pr_warn("Failed to pending transaction\n");
+
+	hclk_err = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_HCLK_INT_STATUS);
+	xclk_err = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_XCLK_INT_STATUS);
+	xclk_ecc_uncor_err_0 = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_XCLK_MEM_ECC_UNCOR_0);
+	xclk_ecc_uncor_err_1 = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_XCLK_MEM_ECC_UNCOR_1);
+	hclk_ecc_uncor_err = YCC_CSR_RD(cfg_bar->vaddr, REG_YCC_HCLK_MEM_ECC_UNCOR);
+
+	if ((hclk_err & ~(YCC_HCLK_TRNG_ERR)) || xclk_err || hclk_ecc_uncor_err) {
+		pr_err("Got uncorrected error, must be reset\n");
+		/*
+		 * Fatal error, as ycc cannot be reset in REE, clear ring data.
+		 */
+		return ycc_fatal_error(ydev);
+	}
+
+	if (xclk_ecc_uncor_err_0 || xclk_ecc_uncor_err_1) {
+		pr_err("Got algorithm ECC error: %x, %x\n",
+			xclk_ecc_uncor_err_0, xclk_ecc_uncor_err_1);
+		return ycc_fatal_error(ydev);
+	}
+
+	/* This has to be queue error. Handling command rings. */
+	for (i = 0; i < YCC_RINGPAIR_NUM; i++) {
+		ring = ydev->rings + i;
+
+		if (ring->type != KERN_RING)
+			continue;
+
+		ring->status = YCC_CSR_RD(ring->csr_vaddr, REG_RING_STATUS);
+		if (ring->status) {
+			pr_err("YCC: Dev: %d, Ring: %d got ring err: %x\n",
+				ydev->id, ring->ring_id, ring->status);
+			spin_lock_bh(&ring->lock);
+			ycc_clear_cmd_ring(ring);
+			spin_unlock_bh(&ring->lock);
+		}
+	}
+
+	/*
+	 * Give HW a chance to process all pending_cmds
+	 * through recovering transactions.
+	 */
+	pci_set_master(ydev->pdev);
+
+	for (i = 0; i < YCC_RINGPAIR_NUM; i++) {
+		ring = ydev->rings + i;
+
+		if (ring->type != KERN_RING || !ring->status)
+			continue;
+
+		ycc_clear_resp_ring(ring);
+	}
+
+	ycc_g_err_unmask(cfg_bar->vaddr);
+	clear_bit(YDEV_STATUS_ERR, &ydev->status);
+	set_bit(YDEV_STATUS_READY, &ydev->status);
 }
 
 static irqreturn_t ycc_g_err_isr(int irq, void *data)

@@ -45,6 +129,9 @@ static irqreturn_t ycc_g_err_isr(int irq, void *data)
 
 	clear_bit(YDEV_STATUS_READY, &ydev->status);
 
+	/* Disable YCC mastering, no new transactions */
+	pci_clear_master(ydev->pdev);
+
 	schedule_work(&ydev->work);
 	return IRQ_HANDLED;
 }

99 lines not on this page · raw message

index eb3e6f9..c3e7cbf 100644
@@ -20,6 +20,9 @@
 #define CMD_INVALID_CONTENT_U8		0x7f
 #define CMD_INVALID_CONTENT_U64		0x7f7f7f7f7f7f7f7fULL
 
+#define MAX_SLEEP_US_PER_CHECK		100   /* every 100us to check register */
+#define MAX_ERROR_RETRY			10000 /* 1s in total */
+
 enum ring_type {
 	FREE_RING,
 	USER_RING,

114 lines not on this page · raw message

index dda82b0..08f9a05 100644
@@ -151,4 +151,7 @@ static inline void ycc_g_err_unmask(void *vaddr)
 	YCC_CSR_WR(vaddr, REG_YCC_DEV_INT_MASK, 0);
 }
 
+int ycc_algorithm_register(void);
+void ycc_algorithm_unregister(void);
+
 #endif

90 lines not on this page · raw message

index c3e7cbf..78ba959 100644
@@ -75,11 +75,20 @@ struct ycc_resp_desc {
 	u8 reserved[6];
 };
 
+struct ycc_skcipher_cmd {
+	u8 cmd_id;
+	u8 mode;
+	u64 sptr:48;
+	u64 dptr:48;
+	u32 dlen;
+	u16 key_idx;	/* key used to decrypt kek */
+	u8 reserved[2];
+	u64 keyptr:48;
+	u8 padding;
+} __packed;
+
 union ycc_real_cmd {
-	/*
-	 * TODO: Real command will implement when
-	 * corresponding algorithm is ready
-	 */
+	struct ycc_skcipher_cmd ske_cmd;
 	u8 padding[32];
 };
 

925 lines not on this page · raw message

646 lines not on this page · raw message

index 6c7b0dc..e3be83ec 100644
@@ -3,6 +3,7 @@
 #define __YCC_ALG_H
 
 #include <crypto/skcipher.h>
+#include <crypto/aead.h>
 
 #include "ycc_ring.h"
 #include "ycc_dev.h"

@@ -70,6 +71,11 @@ enum ycc_ske_alg_mode {
 enum ycc_cmd_id {
 	YCC_CMD_SKE_ENC = 0x23,
 	YCC_CMD_SKE_DEC,
+
+	YCC_CMD_GCM_ENC = 0x25,
+	YCC_CMD_GCM_DEC,
+	YCC_CMD_CCM_ENC,
+	YCC_CMD_CCM_DEC, /* 0x28 */
 };
 
 struct ycc_crypto_ctx {

@@ -92,8 +98,10 @@ struct ycc_crypto_req {
 	dma_addr_t key_paddr;
 
 	struct ycc_cmd_desc desc;
-	struct skcipher_request *ske_req;
-	struct skcipher_request ske_subreq;
+	union {
+		struct skcipher_request *ske_req;
+		struct aead_request *aead_req;
+	};
 
 	void *src_vaddr;
 	dma_addr_t src_paddr;

@@ -105,10 +113,18 @@ struct ycc_crypto_req {
 	int aad_offset;
 	struct ycc_crypto_ctx *ctx;
 	u8 last_block[16]; /* used to store iv out when decrypt */
+
+	/* soft request for fallback, keep at the end */
+	union {
+		struct skcipher_request ske_subreq;
+		struct aead_request aead_subreq;
+	};
 };
 
 #define YCC_DEV(ctx)		(&(ctx)->ring->ydev->pdev->dev)
 
 int ycc_sym_register(void);
 void ycc_sym_unregister(void);
+int ycc_aead_register(void);
+void ycc_aead_unregister(void);
 #endif

index 79c1e18..cd583d7 100644
@@ -94,8 +94,14 @@ int ycc_algorithm_register(void)
 	if (ret)
 		goto err;
 
+	ret = ycc_aead_register();
+	if (ret)
+		goto unregister_sym;
+
 	return 0;
 
+unregister_sym:
+	ycc_sym_unregister();
 err:
 	atomic_dec(&ycc_algs_refcnt);
 	return ret;

@@ -109,6 +115,7 @@ void ycc_algorithm_unregister(void)
 	if (atomic_dec_return(&ycc_algs_refcnt))
 		return;
 
+	ycc_aead_unregister();
 	ycc_sym_unregister();
 }
 

index 78ba959..2caa9e0 100644
@@ -87,8 +87,22 @@ struct ycc_skcipher_cmd {
 	u8 padding;
 } __packed;
 
+struct ycc_aead_cmd {
+	u8 cmd_id;
+	u8 mode;
+	u64 sptr:48;	/* include aad + payload */
+	u64 dptr:48;	/* encrypted/decrypted + tag */
+	u32 dlen;	/* data size */
+	u16 key_idx;
+	u16 kek_idx;
+	u64 keyptr:48;
+	u16 aadlen;
+	u8 taglen;	/* authenc size */
+} __packed;
+
 union ycc_real_cmd {
 	struct ycc_skcipher_cmd ske_cmd;
+	struct ycc_aead_cmd aead_cmd;
 	u8 padding[32];
 };
 

60 lines not on this page · raw message

index cd583d7..4a4d5cb 100644
@@ -98,8 +98,14 @@ int ycc_algorithm_register(void)
 	if (ret)
 		goto unregister_sym;
 
+	ret = ycc_pke_register();
+	if (ret)
+		goto unregister_aead;
+
 	return 0;
 
+unregister_aead:
+	ycc_aead_unregister();
 unregister_sym:
 	ycc_sym_unregister();
 err:

@@ -115,6 +121,7 @@ void ycc_algorithm_unregister(void)
 	if (atomic_dec_return(&ycc_algs_refcnt))
 		return;
 
+	ycc_pke_unregister();
 	ycc_aead_unregister();
 	ycc_sym_unregister();
 }

696 lines not on this page · raw message

index 2caa9e0..c47fc18 100644
@@ -100,9 +100,32 @@ struct ycc_aead_cmd {
 	u8 taglen;	/* authenc size */
 } __packed;
 
+struct ycc_rsa_enc_cmd {
+	u8 cmd_id;
+	u64 sptr:48;
+	u16 key_id;
+	u64 keyptr:48;	/* public key e+n Bytes */
+	u16 elen;	/* bits not byte */
+	u16 nlen;
+	u64 dptr:48;
+} __packed;
+
+struct ycc_rsa_dec_cmd {
+	u8 cmd_id;
+	u64 sptr:48;
+	u16 key_id;
+	u16 kek_id;
+	u64 keyptr:48;	/* private key e + pin + d + n */
+	u16 elen;	/* bits not byte */
+	u16 nlen;
+	u64 dptr:48;
+} __packed;
+
 union ycc_real_cmd {
 	struct ycc_skcipher_cmd ske_cmd;
 	struct ycc_aead_cmd aead_cmd;
+	struct ycc_rsa_enc_cmd rsa_enc_cmd;
+	struct ycc_rsa_dec_cmd rsa_dec_cmd;
 	u8 padding[32];
 };
 

new file mode 100644
index 00000000..1fd15c1
@@ -0,0 +1,38 @@
+/*
+ * Automatically generated by asn1_compiler.  Do not edit
+ *
+ * ASN.1 parser for sm2signature
+ */
+#include <linux/asn1_ber_bytecode.h>
+#include "sm2signature_asn1.h"
+
+enum sm2signature_actions {
+	ACT_sm2_get_signature_r = 0,
+	ACT_sm2_get_signature_s = 1,
+	NR__sm2signature_actions = 2
+};
+
+static const asn1_action_t sm2signature_action_table[NR__sm2signature_actions] = {
+	[0] = sm2_get_signature_r,
+	[1] = sm2_get_signature_s,
+};
+
+static const unsigned char sm2signature_machine[] = {
+	// Sm2Signature
+	[0] = ASN1_OP_MATCH,
+	[1] = _tag(UNIV, CONS, SEQ),
+	[2] =  ASN1_OP_MATCH_ACT,		// sig_r
+	[3] =  _tag(UNIV, PRIM, INT),
+	[4] =  _action(ACT_sm2_get_signature_r),
+	[5] =  ASN1_OP_MATCH_ACT,		// sig_s
+	[6] =  _tag(UNIV, PRIM, INT),
+	[7] =  _action(ACT_sm2_get_signature_s),
+	[8] = ASN1_OP_END_SEQ,
+	[9] = ASN1_OP_COMPLETE,
+};
+
+const struct asn1_decoder sm2signature_decoder = {
+	.machine = sm2signature_machine,
+	.machlen = sizeof(sm2signature_machine),
+	.actions = sm2signature_action_table,
+};

new file mode 100644
index 00000000..192c9e1
@@ -0,0 +1,13 @@
+/*
+ * Automatically generated by asn1_compiler.  Do not edit
+ *
+ * ASN.1 parser for sm2signature
+ */
+#include <linux/asn1_decoder.h>
+
+extern const struct asn1_decoder sm2signature_decoder;
+
+extern int sm2_get_signature_r(void *context, size_t hdrlen,
+		unsigned char tag, const void *value, size_t vlen);
+extern int sm2_get_signature_s(void *context, size_t hdrlen,
+		unsigned char tag, const void *value, size_t vlen);

index 6a13230a..26323a8 100644
@@ -77,6 +77,8 @@ enum ycc_cmd_id {
 	YCC_CMD_CCM_ENC,
 	YCC_CMD_CCM_DEC, /* 0x28 */
 
+	YCC_CMD_SM2_VERIFY = 0x47,
+
 	YCC_CMD_RSA_ENC = 0x83,
 	YCC_CMD_RSA_DEC,
 	YCC_CMD_RSA_CRT_DEC,

273 lines not on this page · raw message

index c47fc18..6dafdc7 100644
@@ -121,11 +121,19 @@ struct ycc_rsa_dec_cmd {
 	u64 dptr:48;
 } __packed;
 
+struct ycc_sm2_verify_cmd {
+	u8 cmd_id;
+	u64 sptr:48;
+	u16 key_id;
+	u64 keyptr:48;
+} __packed;
+
 union ycc_real_cmd {
 	struct ycc_skcipher_cmd ske_cmd;
 	struct ycc_aead_cmd aead_cmd;
 	struct ycc_rsa_enc_cmd rsa_enc_cmd;
 	struct ycc_rsa_dec_cmd rsa_dec_cmd;
+	struct ycc_sm2_verify_cmd sm2_verify_cmd;
 	u8 padding[32];
 };