CRC64 library rework and x86 CRC optimization

From: Eric Biggers <[email protected]>

Following what was done for the CRC32 and CRC-T10DIF library functions, get rid of the pointless use of the crypto API and make crc64_rocksoft_update() call into the library directly. This is faster and simpler.

Remove crc64_rocksoft() (the version of the function that did not take a 'crc' argument) since it is unused.

Signed-off-by Eric Biggers <[email protected]>

---
 block/Kconfig         |   2 +-
 include/linux/crc64.h |  13 ++++-
 lib/Kconfig           |   9 ---
 lib/Makefile          |   1 -
 lib/crc64-rocksoft.c  | 126 ------------------------------------------
 lib/crc64.c           |   7 ---
 6 files changed, 12 insertions(+), 146 deletions(-)
 delete mode 100644 lib/crc64-rocksoft.c

block/Kconfig +1 -1

index 5b623b876d3b..df8973bc0539 100644
@@ -61,11 +61,11 @@ config BLK_DEV_BSGLIB
          If unsure, say N.
 
 config BLK_DEV_INTEGRITY
        bool "Block layer data integrity support"
        select CRC_T10DIF
-       select CRC64_ROCKSOFT
+       select CRC64
        help
        Some storage devices allow extra information to be
        stored/retrieved to help protect the data.  The block layer
        data integrity option provides hooks which can be used by
        filesystems to ensure better data integrity.

include/linux/crc64.h +11 -2

index e044c60d1e61..0a595b272166 100644
@@ -10,9 +10,18 @@
 #define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
 
 u64 __pure crc64_be(u64 crc, const void *p, size_t len);
 u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
 
-u64 crc64_rocksoft(const unsigned char *buffer, size_t len);
-u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len);
+/**
+ * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
+ * @crc: seed value for computation. 0 for a new CRC calculation, or the
+ *      previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
+{
+       return crc64_rocksoft_generic(crc, p, len);
+}
 
 #endif /* _LINUX_CRC64_H */

lib/Kconfig +0 -9

index dccb61b7d698..da07fd39cf97 100644
@@ -166,19 +166,10 @@ config ARCH_HAS_CRC_T10DIF
 
 config CRC_T10DIF_ARCH
        tristate
        default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS
 
-config CRC64_ROCKSOFT
-       tristate "CRC calculation for the Rocksoft model CRC64"
-       select CRC64
-       select CRYPTO
-       select CRYPTO_CRC64_ROCKSOFT
-       help
-         This option provides a CRC64 API to a registered crypto driver.
-         This is used with the block layer's data integrity subsystem.
-
 config CRC_ITU_T
        tristate "CRC ITU-T V.41 functions"
        help
          This option is provided for the case where no in-kernel-tree
          modules require CRC ITU-T V.41 functions, but a module built outside

lib/Makefile +0 -1

index f1c6e9d76a7c..518018b2a5d4 100644
@@ -164,11 +164,10 @@ obj-$(CONFIG_CRC_ITU_T)   += crc-itu-t.o
 obj-$(CONFIG_CRC32)    += crc32.o
 obj-$(CONFIG_CRC64)     += crc64.o
 obj-$(CONFIG_CRC4)     += crc4.o
 obj-$(CONFIG_CRC7)     += crc7.o
 obj-$(CONFIG_CRC8)     += crc8.o
-obj-$(CONFIG_CRC64_ROCKSOFT) += crc64-rocksoft.o
 obj-$(CONFIG_XXHASH)   += xxhash.o
 obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o
 
 obj-$(CONFIG_842_COMPRESS) += 842/
 obj-$(CONFIG_842_DECOMPRESS) += 842/

lib/crc64-rocksoft.c deleted +0 -126

deleted file mode 100644
index fc9ae0da5df7..000000000000
@@ -1,126 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/types.h>
-#include <linux/module.h>
-#include <linux/crc64.h>
-#include <linux/err.h>
-#include <linux/init.h>
-#include <crypto/hash.h>
-#include <crypto/algapi.h>
-#include <linux/static_key.h>
-#include <linux/notifier.h>
-
-static struct crypto_shash __rcu *crc64_rocksoft_tfm;
-static DEFINE_STATIC_KEY_TRUE(crc64_rocksoft_fallback);
-static DEFINE_MUTEX(crc64_rocksoft_mutex);
-static struct work_struct crc64_rocksoft_rehash_work;
-
-static int crc64_rocksoft_notify(struct notifier_block *self, unsigned long val, void *data)
-{
-       struct crypto_alg *alg = data;
-
-       if (val != CRYPTO_MSG_ALG_LOADED ||
-           strcmp(alg->cra_name, CRC64_ROCKSOFT_STRING))
-               return NOTIFY_DONE;
-
-       schedule_work(&crc64_rocksoft_rehash_work);
-       return NOTIFY_OK;
-}
-
-static void crc64_rocksoft_rehash(struct work_struct *work)
-{
-       struct crypto_shash *new, *old;
-
-       mutex_lock(&crc64_rocksoft_mutex);
-       old = rcu_dereference_protected(crc64_rocksoft_tfm,
-                                       lockdep_is_held(&crc64_rocksoft_mutex));
-       new = crypto_alloc_shash(CRC64_ROCKSOFT_STRING, 0, 0);
-       if (IS_ERR(new)) {
-               mutex_unlock(&crc64_rocksoft_mutex);
-               return;
-       }
-       rcu_assign_pointer(crc64_rocksoft_tfm, new);
-       mutex_unlock(&crc64_rocksoft_mutex);
-
-       if (old) {
-               synchronize_rcu();
-               crypto_free_shash(old);
-       } else {
-               static_branch_disable(&crc64_rocksoft_fallback);
-       }
-}
-
-static struct notifier_block crc64_rocksoft_nb = {
-       .notifier_call = crc64_rocksoft_notify,
-};
-
-u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len)
-{
-       struct {
-               struct shash_desc shash;
-               u64 crc;
-       } desc;
-       int err;
-
-       if (static_branch_unlikely(&crc64_rocksoft_fallback))
-               return crc64_rocksoft_generic(crc, buffer, len);
-
-       rcu_read_lock();
-       desc.shash.tfm = rcu_dereference(crc64_rocksoft_tfm);
-       desc.crc = crc;
-       err = crypto_shash_update(&desc.shash, buffer, len);
-       rcu_read_unlock();
-
-       BUG_ON(err);
-
-       return desc.crc;
-}
-EXPORT_SYMBOL_GPL(crc64_rocksoft_update);
-
-u64 crc64_rocksoft(const unsigned char *buffer, size_t len)
-{
-       return crc64_rocksoft_update(0, buffer, len);
-}
-EXPORT_SYMBOL_GPL(crc64_rocksoft);
-
-static int __init crc64_rocksoft_mod_init(void)
-{
-       INIT_WORK(&crc64_rocksoft_rehash_work, crc64_rocksoft_rehash);
-       crypto_register_notifier(&crc64_rocksoft_nb);
-       crc64_rocksoft_rehash(&crc64_rocksoft_rehash_work);
-       return 0;
-}
-
-static void __exit crc64_rocksoft_mod_fini(void)
-{
-       crypto_unregister_notifier(&crc64_rocksoft_nb);
-       cancel_work_sync(&crc64_rocksoft_rehash_work);
-       crypto_free_shash(rcu_dereference_protected(crc64_rocksoft_tfm, 1));
-}
-
-module_init(crc64_rocksoft_mod_init);
-module_exit(crc64_rocksoft_mod_fini);
-
-static int crc64_rocksoft_transform_show(char *buffer, const struct kernel_param *kp)
-{
-       struct crypto_shash *tfm;
-       int len;
-
-       if (static_branch_unlikely(&crc64_rocksoft_fallback))
-               return sprintf(buffer, "fallback\n");
-
-       rcu_read_lock();
-       tfm = rcu_dereference(crc64_rocksoft_tfm);
-       len = snprintf(buffer, PAGE_SIZE, "%s\n",
-                      crypto_shash_driver_name(tfm));
-       rcu_read_unlock();
-
-       return len;
-}
-
-module_param_call(transform, NULL, crc64_rocksoft_transform_show, NULL, 0444);
-
-MODULE_AUTHOR("Keith Busch <[email protected]>");
-MODULE_DESCRIPTION("Rocksoft model CRC64 calculation (library API)");
-MODULE_LICENSE("GPL");
-MODULE_SOFTDEP("pre: crc64");

lib/crc64.c +0 -8

index 61ae8dfb6a1c..b5136fb4c199 100644
@@ -61,17 +61,10 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
 
        return crc;
 }
 EXPORT_SYMBOL_GPL(crc64_be);
 
-/**
- * crc64_rocksoft_generic - Calculate bitwise Rocksoft CRC64
- * @crc: seed value for computation. 0 for a new CRC calculation, or the
- *      previous crc64 value if computing incrementally.
- * @p: pointer to buffer over which CRC64 is run
- * @len: length of buffer @p
- */
 u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
 {
        const unsigned char *_p = p;
        size_t i;
 
--

2.48.1

From: Eric Biggers <[email protected]>

Remove crc64-rocksoft from the crypto API. It has no known user now that the lib is no longer built on top of it. It was also added much more recently than the longstanding crc32 and crc32c. Unlike crc32 and crc32c, crc64-rocksoft is also not mentioned in the dm-integrity documentation and there are no references to it in anywhere in the cryptsetup git repo, so it is unlikely to have any user there either.

Also, this CRC variant is named incorrectly; it has nothing to do with Rocksoft and should be called crc64-nvme. That is yet another reason to remove it from the crypto API; we would not want anyone to start depending on the current incorrect algorithm name of crc64-rocksoft.

Note that this change temporarily makes this CRC variant not be covered by any tests, as previously it was relying on the crypto self-tests. This will be fixed by adding this CRC variant to crc_kunit.

Signed-off-by Eric Biggers <[email protected]>

---
 crypto/Kconfig                  | 11 ----
 crypto/Makefile                 |  1 -
 crypto/crc64_rocksoft_generic.c | 89 ---------------------------------
 crypto/testmgr.c                |  7 ---
 crypto/testmgr.h                | 12 -----
 include/linux/crc64.h           |  2 -
 6 files changed, 122 deletions(-)
 delete mode 100644 crypto/crc64_rocksoft_generic.c

crypto/Kconfig +0 -11

index 74ae5f52b784..9ffb59b1aac3 100644
@@ -1088,21 +1088,10 @@ config CRYPTO_CRCT10DIF
        help
          CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF)
 
          CRC algorithm used by the SCSI Block Commands standard.
 
-config CRYPTO_CRC64_ROCKSOFT
-       tristate "CRC64 based on Rocksoft Model algorithm"
-       depends on CRC64
-       select CRYPTO_HASH
-       help
-         CRC64 CRC algorithm based on the Rocksoft Model CRC Algorithm
-
-         Used by the NVMe implementation of T10 DIF (BLK_DEV_INTEGRITY)
-
-         See https://zlib.net/crc_v3.txt
-
 endmenu
 
 menu "Compression"
 
 config CRYPTO_DEFLATE

crypto/Makefile +0 -1

index f67e853c4690..d3b79b8c9022 100644
@@ -155,11 +155,10 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o
 obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o
 CFLAGS_crc32c_generic.o += -DARCH=$(ARCH)
 CFLAGS_crc32_generic.o += -DARCH=$(ARCH)
 obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_generic.o
 CFLAGS_crct10dif_generic.o += -DARCH=$(ARCH)
-obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o
 obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o
 obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o
 obj-$(CONFIG_CRYPTO_LZ4) += lz4.o
 obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o
 obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o

crypto/crc64_rocksoft_generic.c deleted +0 -89

deleted file mode 100644
index ce0f3059b912..000000000000
@@ -1,89 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/crc64.h>
-#include <linux/module.h>
-#include <crypto/internal/hash.h>
-#include <linux/unaligned.h>
-
-static int chksum_init(struct shash_desc *desc)
-{
-       u64 *crc = shash_desc_ctx(desc);
-
-       *crc = 0;
-
-       return 0;
-}
-
-static int chksum_update(struct shash_desc *desc, const u8 *data,
-                        unsigned int length)
-{
-       u64 *crc = shash_desc_ctx(desc);
-
-       *crc = crc64_rocksoft_generic(*crc, data, length);
-
-       return 0;
-}
-
-static int chksum_final(struct shash_desc *desc, u8 *out)
-{
-       u64 *crc = shash_desc_ctx(desc);
-
-       put_unaligned_le64(*crc, out);
-       return 0;
-}
-
-static int __chksum_finup(u64 crc, const u8 *data, unsigned int len, u8 *out)
-{
-       crc = crc64_rocksoft_generic(crc, data, len);
-       put_unaligned_le64(crc, out);
-       return 0;
-}
-
-static int chksum_finup(struct shash_desc *desc, const u8 *data,
-                       unsigned int len, u8 *out)
-{
-       u64 *crc = shash_desc_ctx(desc);
-
-       return __chksum_finup(*crc, data, len, out);
-}
-
-static int chksum_digest(struct shash_desc *desc, const u8 *data,
-                        unsigned int length, u8 *out)
-{
-       return __chksum_finup(0, data, length, out);
-}
-
-static struct shash_alg alg = {
-       .digestsize     =       sizeof(u64),
-       .init           =       chksum_init,
-       .update         =       chksum_update,
-       .final          =       chksum_final,
-       .finup          =       chksum_finup,
-       .digest         =       chksum_digest,
-       .descsize       =       sizeof(u64),
-       .base           =       {
-               .cra_name               =       CRC64_ROCKSOFT_STRING,
-               .cra_driver_name        =       "crc64-rocksoft-generic",
-               .cra_priority           =       200,
-               .cra_blocksize          =       1,
-               .cra_module             =       THIS_MODULE,
-       }
-};
-
-static int __init crc64_rocksoft_init(void)
-{
-       return crypto_register_shash(&alg);
-}
-
-static void __exit crc64_rocksoft_exit(void)
-{
-       crypto_unregister_shash(&alg);
-}
-
-module_init(crc64_rocksoft_init);
-module_exit(crc64_rocksoft_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Rocksoft model CRC64 calculation.");
-MODULE_ALIAS_CRYPTO("crc64-rocksoft");
-MODULE_ALIAS_CRYPTO("crc64-rocksoft-generic");

crypto/testmgr.c +0 -7

index e61490ba4095..0c1c3a6453b6 100644
@@ -4757,17 +4757,10 @@ static const struct alg_test_desc alg_test_descs[] = {
                .test = alg_test_crc32c,
                .fips_allowed = 1,
                .suite = {
                        .hash = __VECS(crc32c_tv_template)
                }
-       }, {
-               .alg = "crc64-rocksoft",
-               .test = alg_test_hash,
-               .fips_allowed = 1,
-               .suite = {
-                       .hash = __VECS(crc64_rocksoft_tv_template)
-               }
        }, {
                .alg = "crct10dif",
                .test = alg_test_hash,
                .fips_allowed = 1,
                .suite = {

crypto/testmgr.h +0 -12

index d754ab997186..4ab05046b734 100644
@@ -6018,22 +6018,10 @@ static const struct hash_testvec rmd160_tv_template[] = {
 };
 
 static const u8 zeroes[4096] = { [0 ... 4095] = 0 };
 static const u8 ones[4096] = { [0 ... 4095] = 0xff };
 
-static const struct hash_testvec crc64_rocksoft_tv_template[] = {
-       {
-               .plaintext      = zeroes,
-               .psize          = 4096,
-               .digest         = "\x4e\xb6\x22\xeb\x67\xd3\x82\x64",
-       }, {
-               .plaintext      = ones,
-               .psize          = 4096,
-               .digest         = "\xac\xa3\xec\x02\x73\xba\xdd\xc0",
-       }
-};
-
 static const struct hash_testvec crct10dif_tv_template[] = {
        {
                .plaintext      = "abc",
                .psize          = 3,
                .digest         = (u8 *)(u16 []){ 0x443b },

include/linux/crc64.h +0 -3

index 0a595b272166..7880aeab69d6 100644
@@ -5,12 +5,10 @@
 #ifndef _LINUX_CRC64_H
 #define _LINUX_CRC64_H
 
 #include <linux/types.h>
 
-#define CRC64_ROCKSOFT_STRING "crc64-rocksoft"
-
 u64 __pure crc64_be(u64 crc, const void *p, size_t len);
 u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
 
 /**
  * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
--

2.48.1

From: Eric Biggers <[email protected]>

This CRC64 variant comes from the NVME NVM Command Set Specification
(https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0e-2024.07.29-Ratified.pdf).

The "Rocksoft Model CRC Algorithm", published in 1993 and available at
https://www.zlib.net/crc_v3.txt, is a generalized CRC algorithm that can calculate any variant of CRC, given a list of parameters such as polynomial, bit order, etc. It is not a CRC variant.

The NVME NVM Command Set Specification has a table that gives the "Rocksoft Model Parameters" for the CRC variant it uses. When support for this CRC variant was added to Linux, this table seems to have been misinterpreted as naming the CRC variant the "Rocksoft" CRC. In fact, the CRC variant is not explicitly named by the NVME spec.

Most implementations of this CRC variant outside Linux have been calling it CRC64-NVME. Therefore, update Linux to match.

While at it, remove the superfluous "update" from the function name, so crc64_rocksoft_update() is now just crc64_nvme(), matching most of the other CRC library functions.

Signed-off-by Eric Biggers <[email protected]>

---
 block/t10-pi.c        |  2 +-
 include/linux/crc64.h | 11 +++++++----
 lib/crc64.c           | 10 +++++-----
 lib/gen_crc64table.c  | 10 +++++-----
 4 files changed, 18 insertions(+), 15 deletions(-)

block/t10-pi.c +1 -1

index 2d05421f0fa5..2577114ff20c 100644
@@ -208,11 +208,11 @@ static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
        }
 }
 
 static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)
 {
-       return cpu_to_be64(crc64_rocksoft_update(crc, data, len));
+       return cpu_to_be64(crc64_nvme(crc, data, len));
 }
 
 static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,
                struct blk_integrity *bi)
 {

include/linux/crc64.h +7 -4

index 7880aeab69d6..17cf5af3e78e 100644
@@ -6,20 +6,23 @@
 #define _LINUX_CRC64_H
 
 #include <linux/types.h>
 
 u64 __pure crc64_be(u64 crc, const void *p, size_t len);
-u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len);
+u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
 
 /**
- * crc64_rocksoft_update - Calculate bitwise Rocksoft CRC64
+ * crc64_nvme - Calculate CRC64-NVME
  * @crc: seed value for computation. 0 for a new CRC calculation, or the
  *      previous crc64 value if computing incrementally.
  * @p: pointer to buffer over which CRC64 is run
  * @len: length of buffer @p
+ *
+ * This computes the CRC64 defined in the NVME NVM Command Set Specification,
+ * *including the bitwise inversion at the beginning and end*.
  */
-static inline u64 crc64_rocksoft_update(u64 crc, const u8 *p, size_t len)
+static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
 {
-       return crc64_rocksoft_generic(crc, p, len);
+       return crc64_nvme_generic(crc, p, len);
 }
 
 #endif /* _LINUX_CRC64_H */

lib/crc64.c +5 -5

index b5136fb4c199..d6f3f245eede 100644
@@ -20,12 +20,12 @@
  * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
  * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
  * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
  * x^7 + x^4 + x + 1
  *
- * crc64rocksoft[256] table is from the Rocksoft specification polynomial
- * defined as,
+ * crc64nvmetable[256] uses the CRC64 polynomial from the NVME NVM Command Set
+ * Specification and uses least-significant-bit first bit order:
  *
  * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 +
  * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 +
  * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1
  *

@@ -61,18 +61,18 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len)
 
        return crc;
 }
 EXPORT_SYMBOL_GPL(crc64_be);
 
-u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len)
+u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
 {
        const unsigned char *_p = p;
        size_t i;
 
        crc = ~crc;
 
        for (i = 0; i < len; i++)
-               crc = (crc >> 8) ^ crc64rocksofttable[(crc & 0xff) ^ *_p++];
+               crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
 
        return ~crc;
 }
-EXPORT_SYMBOL_GPL(crc64_rocksoft_generic);
+EXPORT_SYMBOL_GPL(crc64_nvme_generic);

lib/gen_crc64table.c +5 -6

index 55e222acd0b8..e05a4230a0a0 100644
@@ -15,14 +15,14 @@
  */
 #include <inttypes.h>
 #include <stdio.h>
 
 #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL
-#define CRC64_ROCKSOFT_POLY 0x9A6C9329AC4BC9B5ULL
+#define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL
 
 static uint64_t crc64_table[256] = {0};
-static uint64_t crc64_rocksoft_table[256] = {0};
+static uint64_t crc64_nvme_table[256] = {0};
 
 static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly)
 {
        uint64_t i, j, c, crc;
 

@@ -80,16 +80,16 @@ static void print_crc64_tables(void)
        printf("#include <linux/types.h>\n");
        printf("#include <linux/cache.h>\n\n");
        printf("static const u64 ____cacheline_aligned crc64table[256] = {\n");
        output_table(crc64_table);
 
-       printf("\nstatic const u64 ____cacheline_aligned crc64rocksofttable[256] = {\n");
-       output_table(crc64_rocksoft_table);
+       printf("\nstatic const u64 ____cacheline_aligned crc64nvmetable[256] = {\n");
+       output_table(crc64_nvme_table);
 }
 
 int main(int argc, char *argv[])
 {
        generate_crc64_table(crc64_table, CRC64_ECMA182_POLY);
-       generate_reflected_crc64_table(crc64_rocksoft_table, CRC64_ROCKSOFT_POLY);
+       generate_reflected_crc64_table(crc64_nvme_table, CRC64_NVME_POLY);
        print_crc64_tables();
        return 0;
 }
--

2.48.1

The NVME NVM Command Set Specification has a table that gives the "Rocksoft Model Parameters" for the CRC variant it uses. When support for this CRC variant was added to Linux, this table seems to have been misinterpreted as naming the CRC variant the "Rocksoft" CRC. In fact, the CRC variant is not explicitly named by the NVME spec.

From: Eric Biggers <[email protected]>

Wire up crc64_nvme() to the new CRC unit test and benchmark.

This replaces and improves on the test coverage that was lost by removing this CRC variant from the crypto API.

Signed-off-by Eric Biggers <[email protected]>

---
 lib/crc_kunit.c | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

lib/crc_kunit.c +29 -2

index 6a61d4b5fd45..1e82fcf9489e 100644
@@ -30,11 +30,12 @@ static size_t test_buflen;
  *     polynomial coefficients in each byte), false if it's a "big endian" CRC
  *     (natural mapping between bits and polynomial coefficients in each byte)
  * @poly: The generator polynomial with the highest-order term omitted.
  *       Bit-reversed if @le is true.
  * @func: The function to compute a CRC.  The type signature uses u64 so that it
- *       can fit any CRC up to CRC-64.
+ *       can fit any CRC up to CRC-64.  The function is expected to *not*
+ *       invert the CRC at the beginning and end.
  * @combine_func: Optional function to combine two CRCs.
  */
 struct crc_variant {
        int bits;
        bool le;

@@ -405,10 +406,35 @@ static void crc64_be_test(struct kunit *test)
 static void crc64_be_benchmark(struct kunit *test)
 {
        crc_benchmark(test, crc64_be_wrapper);
 }
 
+/* crc64_nvme */
+
+static u64 crc64_nvme_wrapper(u64 crc, const u8 *p, size_t len)
+{
+       /* The inversions that crc64_nvme() does have to be undone here. */
+       return ~crc64_nvme(~crc, p, len);
+}
+
+static const struct crc_variant crc_variant_crc64_nvme = {
+       .bits = 64,
+       .le = true,
+       .poly = 0x9a6c9329ac4bc9b5,
+       .func = crc64_nvme_wrapper,
+};
+
+static void crc64_nvme_test(struct kunit *test)
+{
+       crc_test(test, &crc_variant_crc64_nvme);
+}
+
+static void crc64_nvme_benchmark(struct kunit *test)
+{
+       crc_benchmark(test, crc64_nvme_wrapper);
+}
+
 static struct kunit_case crc_test_cases[] = {
        KUNIT_CASE(crc16_test),
        KUNIT_CASE(crc16_benchmark),
        KUNIT_CASE(crc_t10dif_test),
        KUNIT_CASE(crc_t10dif_benchmark),

@@ -418,10 +444,12 @@ static struct kunit_case crc_test_cases[] = {
        KUNIT_CASE(crc32_be_benchmark),
        KUNIT_CASE(crc32c_test),
        KUNIT_CASE(crc32c_benchmark),
        KUNIT_CASE(crc64_be_test),
        KUNIT_CASE(crc64_be_benchmark),
+       KUNIT_CASE(crc64_nvme_test),
+       KUNIT_CASE(crc64_nvme_benchmark),
        {},
 };
 
 static struct kunit_suite crc_test_suite = {
        .name = "crc",
--

2.48.1

From: Eric Biggers <[email protected]>

Add support for architecture-optimized implementations of the CRC64 library functions, following the approach taken for the CRC32 and CRC-T10DIF library functions.

Also take the opportunity to tweak the function prototypes:
- Use 'const void *' for the lib entry points (since this is easier for users) but 'const u8 *' for the underlying arch and generic functions (since this is easier for the implementations of these functions).
- Don't bother with __pure. It's an unusual optimization that doesn't help properly written code. It's a weird quirk we can do without.

Signed-off-by Eric Biggers <[email protected]>

---
 include/linux/crc64.h | 26 ++++++++++++++++++++++----
 lib/Kconfig           |  7 +++++++
 lib/crc64.c           | 36 ++++++++----------------------------
 3 files changed, 37 insertions(+), 32 deletions(-)

include/linux/crc64.h +22 -4

index 17cf5af3e78e..41de30b907df 100644
@@ -5,12 +5,28 @@
 #ifndef _LINUX_CRC64_H
 #define _LINUX_CRC64_H
 
 #include <linux/types.h>
 
-u64 __pure crc64_be(u64 crc, const void *p, size_t len);
-u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
+u64 crc64_be_arch(u64 crc, const u8 *p, size_t len);
+u64 crc64_be_generic(u64 crc, const u8 *p, size_t len);
+u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len);
+u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len);
+
+/**
+ * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
+ * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
+ *       or the previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+static inline u64 crc64_be(u64 crc, const void *p, size_t len)
+{
+       if (IS_ENABLED(CONFIG_CRC64_ARCH))
+               return crc64_be_arch(crc, p, len);
+       return crc64_be_generic(crc, p, len);
+}
 
 /**
  * crc64_nvme - Calculate CRC64-NVME
  * @crc: seed value for computation. 0 for a new CRC calculation, or the
  *      previous crc64 value if computing incrementally.

@@ -18,11 +34,13 @@ u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len);
  * @len: length of buffer @p
  *
  * This computes the CRC64 defined in the NVME NVM Command Set Specification,
  * *including the bitwise inversion at the beginning and end*.
  */
-static inline u64 crc64_nvme(u64 crc, const u8 *p, size_t len)
+static inline u64 crc64_nvme(u64 crc, const void *p, size_t len)
 {
-       return crc64_nvme_generic(crc, p, len);
+       if (IS_ENABLED(CONFIG_CRC64_ARCH))
+               return ~crc64_nvme_arch(~crc, p, len);
+       return ~crc64_nvme_generic(~crc, p, len);
 }
 
 #endif /* _LINUX_CRC64_H */

lib/Kconfig +7 -0

index da07fd39cf97..67bbf4f64dd9 100644
@@ -199,10 +199,17 @@ config CRC64
          This option is provided for the case where no in-kernel-tree
          modules require CRC64 functions, but a module built outside
          the kernel tree does. Such modules that use library CRC64
          functions require M here.
 
+config ARCH_HAS_CRC64
+       bool
+
+config CRC64_ARCH
+       tristate
+       default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS
+
 config CRC4
        tristate "CRC4 functions"
        help
          This option is provided for the case where no in-kernel-tree
          modules require CRC4 functions, but a module built outside

lib/crc64.c +8 -29

index d6f3f245eede..5b1b17057f0a 100644
@@ -39,40 +39,20 @@
 #include "crc64table.h"
 
 MODULE_DESCRIPTION("CRC64 calculations");
 MODULE_LICENSE("GPL v2");
 
-/**
- * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64
- * @crc: seed value for computation. 0 or (u64)~0 for a new CRC calculation,
- *       or the previous crc64 value if computing incrementally.
- * @p: pointer to buffer over which CRC64 is run
- * @len: length of buffer @p
- */
-u64 __pure crc64_be(u64 crc, const void *p, size_t len)
+u64 crc64_be_generic(u64 crc, const u8 *p, size_t len)
 {
-       size_t i, t;
-
-       const unsigned char *_p = p;
-
-       for (i = 0; i < len; i++) {
-               t = ((crc >> 56) ^ (*_p++)) & 0xFF;
-               crc = crc64table[t] ^ (crc << 8);
-       }
-
+       while (len--)
+               crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++];
        return crc;
 }
-EXPORT_SYMBOL_GPL(crc64_be);
+EXPORT_SYMBOL_GPL(crc64_be_generic);
 
-u64 __pure crc64_nvme_generic(u64 crc, const void *p, size_t len)
+u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len)
 {
-       const unsigned char *_p = p;
-       size_t i;
-
-       crc = ~crc;
-
-       for (i = 0; i < len; i++)
-               crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
-
-       return ~crc;
+       while (len--)
+               crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++];
+       return crc;
 }
 EXPORT_SYMBOL_GPL(crc64_nvme_generic);
--

2.48.1

Patch 11 wires up the same optimization to crc64_be() and crc64_nvme() (a.k.a. the old "crc64_rocksoft") which previously were unoptimized,

improving the performance of those CRC functions by as much as 100x.

Patches 1-5 rework the CRC64 library along the lines of what I did for CRC32 and CRC-T10DIF in 6.14. They add direct support for architecture-specific optimizations, fix the naming of the NVME CRC64 variant, and eliminate a pointless use of the crypto API.

Patches 6-10 replace the existing x86 PCLMULQDQ optimized CRC code with new code that is shared among the different CRC variants and also adds VPCLMULQDQ support, greatly improving performance on recent CPUs. Patch 11 wires up the same optimization to crc64_be() and crc64_nvme() (a.k.a. the old "crc64_rocksoft") which previously were unoptimized, improving the performance of those CRC functions by as much as 100x. crc64_be is used by bcachefs, and crc64_nvme is used by blk-integrity.

This patchset applies to commit 72deda0abee6e7 and is also available at:

git fetch https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-x86-v2

This is the next major set of CRC library improvements, targeting 6.15.

Patches 1-5 rework the CRC64 library along the lines of what I did for CRC32 and CRC-T10DIF in 6.14. They add direct support for architecture-specific optimizations, fix the naming of the NVME CRC64 variant, and eliminate a pointless use of the crypto API.

Patches 6-10 replace the existing x86 PCLMULQDQ optimized CRC code with new code that is shared among the different CRC variants and also adds VPCLMULQDQ support, greatly improving performance on recent CPUs. Patch 11 wires up the same optimization to crc64_be() and crc64_nvme() (a.k.a. the old "crc64_rocksoft") which previously were unoptimized, improving the performance of those CRC functions by as much as 100x. crc64_be is used by bcachefs, and crc64_nvme is used by blk-integrity.