Skip to contentSkip to conversation

unicode: Expose available encodings in sysfs

PATCH Superseded Gabriel Krisman Bertazi 3 participants 4 messages Apr 13, 2020 – Apr 14, 2020

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 unicode: Expose available encodings in sysfs

A filesystem configuration utility has no way to detect which filename encodings are supported by the running kernel. This means, for instance, mkfs has no way to tell if the generated filesystem will be mountable in the current kernel or not. Also, users have no easy way to know if they can update the encoding in their filesystems and still have something functional in the end.

This exposes details of the encodings available in the unicode subsystem, to fill that gap.

---
Changes since v1:
- Make init functions static. (lkp)

 Documentation/ABI/testing/sysfs-fs-unicode | 13 +++++
 fs/unicode/utf8-core.c                     | 64 ++++++++++++++++++++++
 fs/unicode/utf8-norm.c                     | 18 ++++++
 fs/unicode/utf8n.h                         |  5 ++
 4 files changed, 100 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-fs-unicode
Documentation/ABI/testing/sysfs-fs-unicode new file +13 -0
new file mode 100644
index 000000000000..15c63367bb8e
@@ -0,0 +1,13 @@
+What:		/sys/fs/unicode/latest
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		The latest version of the Unicode Standard supported by
+		this kernel
+
+What:		/sys/fs/unicode/encodings
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		List of encodings and corresponding versions supported
+		by this kernel
fs/unicode/utf8-core.c +64 -0

74 lines not on this page · raw message

fs/unicode/utf8-norm.c +18 -0
index 1d2d2e5b906a..f9ebba89a138 100644
@@ -35,6 +35,24 @@ int utf8version_latest(void)
 }
 EXPORT_SYMBOL(utf8version_latest);
 
+int utf8version_list(char *buf, int len)
+{
+	int i = ARRAY_SIZE(utf8agetab) - 1;
+	int ret = 0;
+
+	/*
+	 * Print most relevant (latest) first.  No filesystem uses
+	 * unicode <= 12.0.0, so don't expose them to userspace.
+	 */
+	for (; utf8agetab[i] >= UNICODE_AGE(12, 0, 0); i--) {
+		ret += snprintf(buf+ret, len-ret, " %d.%d.%d",
+				UNICODE_AGE_MAJ(utf8agetab[i]),
+				UNICODE_AGE_MIN(utf8agetab[i]),
+				UNICODE_AGE_REV(utf8agetab[i]));
+	}
+	return ret;
+}
+
 /*
  * UTF-8 valid ranges.
  *
fs/unicode/utf8n.h +5 -0
index 0acd530c2c79..5dea2c4af1f3 100644
@@ -21,9 +21,14 @@
 	 ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) |	\
 	 ((unsigned int)(REV)))
 
+#define UNICODE_AGE_MAJ(x) ((x) >> UNICODE_MAJ_SHIFT & 0xff)
+#define UNICODE_AGE_MIN(x) ((x) >> UNICODE_MIN_SHIFT & 0xff)
+#define UNICODE_AGE_REV(x) ((x) & 0xff)
+
 /* Highest unicode version supported by the data tables. */
 extern int utf8version_is_supported(u8 maj, u8 min, u8 rev);
 extern int utf8version_latest(void);
+extern int utf8version_list(char *buf, int len);
 
 /*
  * Look for the correct const struct utf8data for a unicode version.
Signature · 1 line
2.26.0
#1 lore raw

Hey Gabriel,

Quoted 31 lines · Gabriel Krisman Bertazi

A filesystem configuration utility has no way to detect which filename encodings are supported by the running kernel. This means, for instance, mkfs has no way to tell if the generated filesystem will be mountable in the current kernel or not. Also, users have no easy way to know if they can update the encoding in their filesystems and still have something functional in the end.

This exposes details of the encodings available in the unicode subsystem, to fill that gap.

---
Changes since v1:
- Make init functions static. (lkp)

 Documentation/ABI/testing/sysfs-fs-unicode | 13 +++++
 fs/unicode/utf8-core.c                     | 64 ++++++++++++++++++++++
 fs/unicode/utf8-norm.c                     | 18 ++++++
 fs/unicode/utf8n.h                         |  5 ++
 4 files changed, 100 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-fs-unicode
Documentation/ABI/testing/sysfs-fs-unicode new file +0 -0
new file mode 100644
index 000000000000..15c63367bb8e

Documentation/ABI/testing/sysfs-fs-unicode line 7

Quoted 7 lines
Documentation/ABI/testing/sysfs-fs-unicode new file +7 -0
@@ -0,0 +1,13 @@
+What:		/sys/fs/unicode/latest
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		The latest version of the Unicode Standard supported by
+		this kernel
+

Missing stop at the end of the sentence?

Documentation/ABI/testing/sysfs-fs-unicode line 13

Quoted 6 lines
Documentation/ABI/testing/sysfs-fs-unicode new file +6 -0
+What:		/sys/fs/unicode/encodings
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		List of encodings and corresponding versions supported
+		by this kernel

Ditto.

Quoted 24 lines
fs/unicode/utf8-core.c +9 -0
index 2a878b739115..b48e13e823a5 100644
@@ -6,6 +6,7 @@
 #include <linux/parser.h>
 #include <linux/errno.h>
 #include <linux/unicode.h>
+#include <linux/fs.h>
 
 #include "utf8n.h"
 
@@ -212,4 +213,67 @@ void utf8_unload(struct unicode_map *um)
 }
 EXPORT_SYMBOL(utf8_unload);
 
+static ssize_t latest_show(struct kobject *kobj,
+			   struct kobj_attribute *attr, char *buf)
+{
+	int l = utf8version_latest();
+
+	return snprintf(buf, PAGE_SIZE, "UTF-8 %d.%d.%d\n", UNICODE_AGE_MAJ(l),
+			UNICODE_AGE_MIN(l), UNICODE_AGE_REV(l));
+

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 233

Quoted 10 lines
fs/unicode/utf8-core.c +10 -0
+}
+static ssize_t encodings_show(struct kobject *kobj,
+			      struct kobj_attribute *attr, char *buf)
+{
+	int n;
+
+	n = snprintf(buf, PAGE_SIZE, "UTF-8:");
+	n += utf8version_list(buf + n, PAGE_SIZE - n);
+	n += snprintf(buf+n, PAGE_SIZE-n, "\n");
+

I was wondering how sysfs-compliant this was,
in terms of one value per attribute.

Although, we do seem to break this on a few cases.

Quoted 33 lines
fs/unicode/utf8-core.c +33 -0
+	return n;
+}
+
+#define UCD_ATTR(x) \
+	static struct kobj_attribute x ## _attr = __ATTR_RO(x)
+
+UCD_ATTR(latest);
+UCD_ATTR(encodings);
+
+static struct attribute *ucd_attrs[] = {
+	&latest_attr.attr,
+	&encodings_attr.attr,
+	NULL,
+};
+static const struct attribute_group ucd_attr_group = {
+	.attrs = ucd_attrs,
+};
+static struct kobject *ucd_root;
+
+static int __init ucd_init(void)
+{
+	int ret;
+
+	ucd_root = kobject_create_and_add("unicode", fs_kobj);
+	if (!ucd_root)
+		return -ENOMEM;
+
+	ret = sysfs_create_group(ucd_root, &ucd_attr_group);
+	if (ret) {
+		kobject_put(ucd_root);
+		ucd_root = NULL;
+		return ret;
+	}

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 276

Quoted 10 lines
fs/unicode/utf8-core.c +10 -0
+
+	return 0;
+}
+
+static void __exit ucd_exit(void)
+{
+	kobject_put(ucd_root);
+}
+
+module_init(ucd_init);

This code is not a module, so how about fs_initcall?

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 278

Quoted 2 lines
fs/unicode/utf8-core.c +2 -0
+module_exit(ucd_exit)
+

I can be wrong, but I see no way to remove it :-)

Thanks,
Ezequiel

#2 lore raw
Quoted 91 lines · Ezequiel Garcia

Hey Gabriel,

Quoted 31 lines · Gabriel Krisman Bertazi

A filesystem configuration utility has no way to detect which filename encodings are supported by the running kernel. This means, for instance, mkfs has no way to tell if the generated filesystem will be mountable in the current kernel or not. Also, users have no easy way to know if they can update the encoding in their filesystems and still have something functional in the end.

This exposes details of the encodings available in the unicode subsystem, to fill that gap.

---
Changes since v1:
- Make init functions static. (lkp)

 Documentation/ABI/testing/sysfs-fs-unicode | 13 +++++
 fs/unicode/utf8-core.c                     | 64 ++++++++++++++++++++++
 fs/unicode/utf8-norm.c                     | 18 ++++++
 fs/unicode/utf8n.h                         |  5 ++
 4 files changed, 100 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-fs-unicode
Documentation/ABI/testing/sysfs-fs-unicode new file +0 -0
new file mode 100644
index 000000000000..15c63367bb8e

Documentation/ABI/testing/sysfs-fs-unicode line 7

Quoted 7 lines
Documentation/ABI/testing/sysfs-fs-unicode new file +7 -0
@@ -0,0 +1,13 @@
+What:		/sys/fs/unicode/latest
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		The latest version of the Unicode Standard supported by
+		this kernel
+

Missing stop at the end of the sentence?

Documentation/ABI/testing/sysfs-fs-unicode line 13

Quoted 6 lines
Documentation/ABI/testing/sysfs-fs-unicode new file +6 -0
+What:		/sys/fs/unicode/encodings
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		List of encodings and corresponding versions supported
+		by this kernel

Ditto.

Quoted 24 lines
fs/unicode/utf8-core.c +9 -0
index 2a878b739115..b48e13e823a5 100644
@@ -6,6 +6,7 @@
 #include <linux/parser.h>
 #include <linux/errno.h>
 #include <linux/unicode.h>
+#include <linux/fs.h>
 
 #include "utf8n.h"
 
@@ -212,4 +213,67 @@ void utf8_unload(struct unicode_map *um)
 }
 EXPORT_SYMBOL(utf8_unload);
 
+static ssize_t latest_show(struct kobject *kobj,
+			   struct kobj_attribute *attr, char *buf)
+{
+	int l = utf8version_latest();
+
+	return snprintf(buf, PAGE_SIZE, "UTF-8 %d.%d.%d\n", UNICODE_AGE_MAJ(l),
+			UNICODE_AGE_MIN(l), UNICODE_AGE_REV(l));
+

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 233

Quoted 10 lines
fs/unicode/utf8-core.c +10 -0
+}
+static ssize_t encodings_show(struct kobject *kobj,
+			      struct kobj_attribute *attr, char *buf)
+{
+	int n;
+
+	n = snprintf(buf, PAGE_SIZE, "UTF-8:");
+	n += utf8version_list(buf + n, PAGE_SIZE - n);
+	n += snprintf(buf+n, PAGE_SIZE-n, "\n");
+

I was wondering how sysfs-compliant this was,
in terms of one value per attribute.

Quoted 1 line

Although, we do seem to break this on a few cases.

I thought about making it more one-value-per-attribute, by considering each tuple (encoding,version) a different object. So the structure would look like:

/sys/fs/unicode/utf-8/<version>/

But it felt completely over-engineered, considering we don't support anything other than utf-8, and I don't see versions having specific attributes. But maybe this way is more future-proof.

Quoted 38 lines
Quoted 38 lines
fs/unicode/utf8-core.c +37 -0
+	return n;
+}
+
+#define UCD_ATTR(x) \
+	static struct kobj_attribute x ## _attr = __ATTR_RO(x)
+
+UCD_ATTR(latest);
+UCD_ATTR(encodings);
+
+static struct attribute *ucd_attrs[] = {
+	&latest_attr.attr,
+	&encodings_attr.attr,
+	NULL,
+};
+static const struct attribute_group ucd_attr_group = {
+	.attrs = ucd_attrs,
+};
+static struct kobject *ucd_root;
+
+static int __init ucd_init(void)
+{
+	int ret;
+
+	ucd_root = kobject_create_and_add("unicode", fs_kobj);
+	if (!ucd_root)
+		return -ENOMEM;
+
+	ret = sysfs_create_group(ucd_root, &ucd_attr_group);
+	if (ret) {
+		kobject_put(ucd_root);
+		ucd_root = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
Quoted 8 lines

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 276

Quoted 6 lines
fs/unicode/utf8-core.c +6 -0
+static void __exit ucd_exit(void)
+{
+	kobject_put(ucd_root);
+}
+
+module_init(ucd_init);

This code is not a module, so how about fs_initcall?

Right. for the record, I'm planing to make part of it build as a module in a future patch to reduce the large footprint of the decoding table on systems that don't care about it. Will fix on v3.

Quoted 8 lines

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 278

Quoted 3 lines
fs/unicode/utf8-core.c +2 -0
+module_exit(ucd_exit)
+

I can be wrong, but I see no way to remove it :-)

Thanks,
Ezequiel

Signature · 1 line
Gabriel Krisman Bertazi
#3 lore raw
In reply to Gabriel Krisman Bertazi
Quoted 67 lines · Gabriel Krisman Bertazi

A filesystem configuration utility has no way to detect which filename encodings are supported by the running kernel. This means, for instance, mkfs has no way to tell if the generated filesystem will be mountable in the current kernel or not. Also, users have no easy way to know if they can update the encoding in their filesystems and still have something functional in the end.

This exposes details of the encodings available in the unicode subsystem, to fill that gap.

---
Changes since v1:
- Make init functions static. (lkp)

 Documentation/ABI/testing/sysfs-fs-unicode | 13 +++++
 fs/unicode/utf8-core.c                     | 64 ++++++++++++++++++++++
 fs/unicode/utf8-norm.c                     | 18 ++++++
 fs/unicode/utf8n.h                         |  5 ++
 4 files changed, 100 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-fs-unicode
Documentation/ABI/testing/sysfs-fs-unicode new file +13 -0
new file mode 100644
index 000000000000..15c63367bb8e
@@ -0,0 +1,13 @@
+What:		/sys/fs/unicode/latest
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		The latest version of the Unicode Standard supported by
+		this kernel
+
+What:		/sys/fs/unicode/encodings
+Date:		April 2020
+Contact:	Gabriel Krisman Bertazi <[email protected]>
+Description:
+		List of encodings and corresponding versions supported
+		by this kernel
fs/unicode/utf8-core.c +8 -0
index 2a878b739115..b48e13e823a5 100644
@@ -6,6 +6,7 @@
 #include <linux/parser.h>
 #include <linux/errno.h>
 #include <linux/unicode.h>
+#include <linux/fs.h>
 
 #include "utf8n.h"
 
@@ -212,4 +213,67 @@ void utf8_unload(struct unicode_map *um)
 }
 EXPORT_SYMBOL(utf8_unload);
 
+static ssize_t latest_show(struct kobject *kobj,
+			   struct kobj_attribute *attr, char *buf)
+{
+	int l = utf8version_latest();
+
+	return snprintf(buf, PAGE_SIZE, "UTF-8 %d.%d.%d\n", UNICODE_AGE_MAJ(l),
+			UNICODE_AGE_MIN(l), UNICODE_AGE_REV(l));

fs/unicode/utf8-core.c void utf8_unload(struct unicode_map *um) · line 232

Quoted 10 lines
fs/unicode/utf8-core.c +10 -0
+
+}
+static ssize_t encodings_show(struct kobject *kobj,
+			      struct kobj_attribute *attr, char *buf)
+{
+	int n;
+
+	n = snprintf(buf, PAGE_SIZE, "UTF-8:");
+	n += utf8version_list(buf + n, PAGE_SIZE - n);
+	n += snprintf(buf+n, PAGE_SIZE-n, "\n");

Spaces before and after the '+' and '-' operators?

	n += snprintf(buf + n, PAGE_SIZE - n, "\n");

Thanks,
Anirudh

Quoted 98 lines
fs/unicode/utf8-core.c +46 -0
+
+	return n;
+}
+
+#define UCD_ATTR(x) \
+	static struct kobj_attribute x ## _attr = __ATTR_RO(x)
+
+UCD_ATTR(latest);
+UCD_ATTR(encodings);
+
+static struct attribute *ucd_attrs[] = {
+	&latest_attr.attr,
+	&encodings_attr.attr,
+	NULL,
+};
+static const struct attribute_group ucd_attr_group = {
+	.attrs = ucd_attrs,
+};
+static struct kobject *ucd_root;
+
+static int __init ucd_init(void)
+{
+	int ret;
+
+	ucd_root = kobject_create_and_add("unicode", fs_kobj);
+	if (!ucd_root)
+		return -ENOMEM;
+
+	ret = sysfs_create_group(ucd_root, &ucd_attr_group);
+	if (ret) {
+		kobject_put(ucd_root);
+		ucd_root = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit ucd_exit(void)
+{
+	kobject_put(ucd_root);
+}
+
+module_init(ucd_init);
+module_exit(ucd_exit)
+
 MODULE_LICENSE("GPL v2");
fs/unicode/utf8-norm.c +18 -0
index 1d2d2e5b906a..f9ebba89a138 100644
@@ -35,6 +35,24 @@ int utf8version_latest(void)
 }
 EXPORT_SYMBOL(utf8version_latest);
 
+int utf8version_list(char *buf, int len)
+{
+	int i = ARRAY_SIZE(utf8agetab) - 1;
+	int ret = 0;
+
+	/*
+	 * Print most relevant (latest) first.  No filesystem uses
+	 * unicode <= 12.0.0, so don't expose them to userspace.
+	 */
+	for (; utf8agetab[i] >= UNICODE_AGE(12, 0, 0); i--) {
+		ret += snprintf(buf+ret, len-ret, " %d.%d.%d",
+				UNICODE_AGE_MAJ(utf8agetab[i]),
+				UNICODE_AGE_MIN(utf8agetab[i]),
+				UNICODE_AGE_REV(utf8agetab[i]));
+	}
+	return ret;
+}
+
 /*
  * UTF-8 valid ranges.
  *
fs/unicode/utf8n.h +5 -0
index 0acd530c2c79..5dea2c4af1f3 100644
@@ -21,9 +21,14 @@
 	 ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) |	\
 	 ((unsigned int)(REV)))
 
+#define UNICODE_AGE_MAJ(x) ((x) >> UNICODE_MAJ_SHIFT & 0xff)
+#define UNICODE_AGE_MIN(x) ((x) >> UNICODE_MIN_SHIFT & 0xff)
+#define UNICODE_AGE_REV(x) ((x) & 0xff)
+
 /* Highest unicode version supported by the data tables. */
 extern int utf8version_is_supported(u8 maj, u8 min, u8 rev);
 extern int utf8version_latest(void);
+extern int utf8version_list(char *buf, int len);
 
 /*
  * Look for the correct const struct utf8data for a unicode version.
Signature · 1 line
2.26.0