Skip to contentSkip to conversation

unicode diacritics support

PATCH In review Samuel Thibault 2 participants 3 messages Aug 8, 2007 – Aug 9, 2007

Subsystems (4)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 unicode diacritics support

Hi,

There have been issues with non-latin1 diacritics and unicode.
http://bugzilla.kernel.org/show_bug.cgi?id=7746

Git 759448f459234bfcf34b82471f0dba77a9aca498 `Kernel utf-8 handling' partly resolved it by adding conversion between diacritics and unicode. The patch below goes further by just turning diacritics into unicode, hence providing better future support. The kbd support can be fetched from
http://bugzilla.kernel.org/attachment.cgi?id=12313

This was tested in all of latin1, latin9, latin2 and unicode with french and czech dead keys.

--

Turn the kernel accent_table into unicode, and extend ioctls KDGKBDIACR and KDSKBDIACR into their equivalents KDGKBDIACRUC and KDSKBDIACR.

New function int conv_uni_to_8bit(u32 uni) for converting unicode into 8bit _input_. No, we don't want to store the translation, as it is potentially sparse and large.

drivers/acorn/char/defkeymap-l7200.c +1 -1
index 9e18ce7..28a5fbc 100644
@@ -346,7 +346,7 @@ char *func_table[MAX_NR_FUNC] = {
 	0,
 };
 
-struct kbdiacr accent_table[MAX_DIACR] = {
+struct kbdiacruc accent_table[MAX_DIACR] = {
 	{'`', 'A', '\300'},	{'`', 'a', '\340'},
 	{'\'', 'A', '\301'},	{'\'', 'a', '\341'},
 	{'^', 'A', '\302'},	{'^', 'a', '\342'},
drivers/char/consolemap.c +16 -6
index 4b3916f..689d820 100644
@@ -670,19 +670,29 @@ void con_protect_unimap(struct vc_data *vc, int rdonly)
 		p->readonly = rdonly;
 }
 
+/*
+ * Always use USER_MAP. These functions are used by the keyboard,
+ * which shouldn't be affected by G0/G1 switching, etc.
+ * If the user map still contains default values, i.e. the
+ * direct-to-font mapping, then assume user is using Latin1.
+ */
 /* may be called during an interrupt */
 u32 conv_8bit_to_uni(unsigned char c)
 {
-	/*
-	 * Always use USER_MAP. This function is used by the keyboard,
-	 * which shouldn't be affected by G0/G1 switching, etc.
-	 * If the user map still contains default values, i.e. the
-	 * direct-to-font mapping, then assume user is using Latin1.
-	 */
 	unsigned short uni = translations[USER_MAP][c];
 	return uni == (0xf000 | c) ? c : uni;
 }
 
+int conv_uni_to_8bit(u32 uni)
+{
+	int c;
+	for (c = 0; c < 0x100; c++)
+		if (translations[USER_MAP][c] == uni ||
+		   (translations[USER_MAP][c] == (c | 0xf000) && uni == c))
+			return c;
+	return -1;
+}
+
 int
 conv_uni_to_pc(struct vc_data *conp, long ucs) 
 {
drivers/char/defkeymap.c_shipped +1 -1
index 453a2f1..0aa419a 100644
@@ -222,7 +222,7 @@ char *func_table[MAX_NR_FUNC] = {
 	NULL,
 };
 
-struct kbdiacr accent_table[MAX_DIACR] = {
+struct kbdiacruc accent_table[MAX_DIACR] = {
 	{'`', 'A', '\300'},	{'`', 'a', '\340'},
 	{'\'', 'A', '\301'},	{'\'', 'a', '\341'},
 	{'^', 'A', '\302'},	{'^', 'a', '\342'},
drivers/char/keyboard.c +25 -10
index 2ce0af1..3bec260 100644
@@ -41,6 +41,7 @@
 #include <linux/sysrq.h>
 #include <linux/input.h>
 #include <linux/reboot.h>
+#include <linux/consolemap.h>
 
 extern void ctrl_alt_del(void);
 
@@ -403,9 +404,12 @@ static unsigned int handle_diacr(struct vc_data *vc, unsigned int ch)
 		return d;
 
 	if (kbd->kbdmode == VC_UNICODE)
-		to_utf8(vc, conv_8bit_to_uni(d));
-	else if (d < 0x100)
-		put_queue(vc, d);
+		to_utf8(vc, d);
+	else {
+		int c = conv_uni_to_8bit(d);
+		if (c != -1)
+			put_queue(vc, c);
+	}
 
 	return ch;
 }
@@ -417,9 +421,12 @@ static void fn_enter(struct vc_data *vc)
 {
 	if (diacr) {
 		if (kbd->kbdmode == VC_UNICODE)
-			to_utf8(vc, conv_8bit_to_uni(diacr));
-		else if (diacr < 0x100)
-			put_queue(vc, diacr);
+			to_utf8(vc, diacr);
+		else {
+			int c = conv_uni_to_8bit(diacr);
+			if (c != -1)
+				put_queue(vc, c);
+		}
 		diacr = 0;
 	}
 	put_queue(vc, 13);
@@ -627,9 +634,12 @@ static void k_unicode(struct vc_data *vc, unsigned int value, char up_flag)
 		return;
 	}
 	if (kbd->kbdmode == VC_UNICODE)
-		to_utf8(vc, conv_8bit_to_uni(value));
-	else if (value < 0x100)
-		put_queue(vc, value);
+		to_utf8(vc, value);
+	else {
+		int c = conv_uni_to_8bit(value);
+		if (c != -1)
+			put_queue(vc, c);
+	}
 }
 
 /*
@@ -646,7 +656,12 @@ static void k_deadunicode(struct vc_data *vc, unsigned int value, char up_flag)
 
 static void k_self(struct vc_data *vc, unsigned char value, char up_flag)
 {
-	k_unicode(vc, value, up_flag);
+	unsigned int uni;
+	if (kbd->kbdmode == VC_UNICODE)
+		uni = value;
+	else
+		uni = conv_8bit_to_uni(value);
+	k_unicode(vc, uni, up_flag);
 }
 
 static void k_dead2(struct vc_data *vc, unsigned char value, char up_flag)
drivers/char/vt_ioctl.c +44 -2

73 lines not on this page · raw message

drivers/s390/char/defkeymap.c +1 -1
index 564baca..389346c 100644
@@ -150,7 +150,7 @@ char *func_table[MAX_NR_FUNC] = {
 	NULL,
 };
 
-struct kbdiacr accent_table[MAX_DIACR] = {
+struct kbdiacruc accent_table[MAX_DIACR] = {
 	{'^', 'c', '\003'},	{'^', 'd', '\004'},
 	{'^', 'z', '\032'},	{'^', '\012', '\000'},
 };
drivers/s390/char/keyboard.c +55 -11

114 lines not on this page · raw message

drivers/s390/char/keyboard.h +2 -2
index f7bf45c..5ccfe9c 100644
@@ -25,9 +25,9 @@ struct kbd_data {
 	unsigned short **key_maps;
 	char **func_table;
 	fn_handler_fn **fn_handler;
-	struct kbdiacr *accent_table;
+	struct kbdiacruc *accent_table;
 	unsigned int accent_table_size;
-	unsigned char diacr;
+	unsigned int diacr;
 	unsigned short sysrq;
 };
 
drivers/tc/lk201-map.c_shipped +1 -1
index a9df8f5..4d2aba5 100644
@@ -225,7 +225,7 @@ char *func_table[MAX_NR_FUNC] = {
 	0,
 };
 
-struct kbdiacr accent_table[MAX_DIACR] = {
+struct kbdiacruc accent_table[MAX_DIACR] = {
 	{'`', 'A', 'À'},	{'`', 'a', 'à'},
 	{'\'', 'A', 'Á'},	{'\'', 'a', 'á'},
 	{'^', 'A', 'Â'},	{'^', 'a', 'â'},
include/linux/consolemap.h +1 -0
index 06b2768..e2bf7e5 100644
@@ -16,4 +16,5 @@ extern u16 inverse_translate(struct vc_data *conp, int glyph, int use_unicode);
 extern unsigned short *set_translate(int m, struct vc_data *vc);
 extern int conv_uni_to_pc(struct vc_data *conp, long ucs);
 extern u32 conv_8bit_to_uni(unsigned char c);
+extern int conv_uni_to_8bit(u32 uni);
 void console_map_init(void);
include/linux/kbd_diacr.h +1 -1
index 1c1a3ff..7274ec6 100644
@@ -2,7 +2,7 @@
 #define _DIACR_H
 #include <linux/kd.h>
 
-extern struct kbdiacr accent_table[];
+extern struct kbdiacruc accent_table[];
 extern unsigned int accent_table_size;
 
 #endif /* _DIACR_H */
include/linux/kd.h +10 -0
index 28be4fb..c91fc0c 100644
@@ -125,6 +125,16 @@ struct kbdiacrs {
 #define KDGKBDIACR      0x4B4A  /* read kernel accent table */
 #define KDSKBDIACR      0x4B4B  /* write kernel accent table */
 
+struct kbdiacruc {
+        __u32 diacr, base, result;
+};
+struct kbdiacrsuc {
+        unsigned int kb_cnt;    /* number of entries in following array */
+	struct kbdiacruc kbdiacruc[256];    /* MAX_DIACR from keyboard.h */
+};
+#define KDGKBDIACRUC    0x4BFA  /* read kernel accent table - UCS */
+#define KDSKBDIACRUC    0x4BFB  /* write kernel accent table - UCS */
+
 struct kbkeycode {
 	unsigned int scancode, keycode;
 };
#1 lore raw
Quoted 13 lines · On Wed, 8 Aug 2007 10:10:10 +0200 Samuel Thibault…

There have been issues with non-latin1 diacritics and unicode.
http://bugzilla.kernel.org/show_bug.cgi?id=7746

Git 759448f459234bfcf34b82471f0dba77a9aca498 `Kernel utf-8 handling' partly resolved it by adding conversion between diacritics and unicode. The patch below goes further by just turning diacritics into unicode, hence providing better future support. The kbd support can be fetched from
http://bugzilla.kernel.org/attachment.cgi?id=12313

This was tested in all of latin1, latin9, latin2 and unicode with french and czech dead keys.

Quoted 8 lines

--

Turn the kernel accent_table into unicode, and extend ioctls KDGKBDIACR and KDSKBDIACR into their equivalents KDGKBDIACRUC and KDSKBDIACR.

New function int conv_uni_to_8bit(u32 uni) for converting unicode into 8bit _input_. No, we don't want to store the translation, as it is potentially sparse and large.

This changes the kernel ABI a bit. The KDGKBDIACR ioctl (and friends) now return different data. What are the implications of this?

#2 lore raw

Hi,

Quoted 9 lines · Andrew Morton, le Thu 09 Aug 2007 12:46:42 -0700, a écrit :
Quoted 6 lines

Turn the kernel accent_table into unicode, and extend ioctls KDGKBDIACR and KDSKBDIACR into their equivalents KDGKBDIACRUC and KDSKBDIACR.

New function int conv_uni_to_8bit(u32 uni) for converting unicode into 8bit _input_. No, we don't want to store the translation, as it is potentially sparse and large.

This changes the kernel ABI a bit. The KDGKBDIACR ioctl (and friends) now return different data.

No, they are still returning data encoded in the consolemap.

What was happening before is that the diacr variable wasn't always encoded in latin1, it was encoded in the consolemap. Now diacr is always unicode, so the diacritical translations now have to be translated into consolemap for the old ioctl() to work as before.

Samuel