Skip to contentSkip to conversation

ensure locale en_US.UTF-8 is available on the system

PATCH In review Frank de Brabander 3 participants 5 messages Dec 6, 2022 – Apr 13, 2023

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 ensure locale en_US.UTF-8 is available on the system

Get rid of the duplicate code and add extra check that the locale en_US.UTF-8 is available on the system. This new helper method is now located right above the method filter_environment() which sets LC_ALL environment variable to 'en_US.UTF-8'.

[YOCTO #10165]

---
 bin/bitbake        |  3 +--
 bin/bitbake-server |  5 +++--
 bin/bitbake-worker |  3 +--
 lib/bb/utils.py    | 16 ++++++++++++++++
 4 files changed, 21 insertions(+), 6 deletions(-)
bin/bitbake +1 -2
index 7cbf88f4..f869eb48 100755
@@ -25,8 +25,7 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 __version__ = "2.2.0"
 
bin/bitbake-server +3 -2
index 825e9d56..454a3919 100755
@@ -12,8 +12,9 @@ warnings.simplefilter("default")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"):
bin/bitbake-worker +1 -2
index 3799b170..3cacdb0c 100755
@@ -24,8 +24,7 @@ import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
lib/bb/utils.py +16 -0
index f4da3563..0df522b3 100644
@@ -13,6 +13,7 @@ import errno
 import logging
 import bb
 import bb.msg
+import locale
 import multiprocessing
 import fcntl
 import importlib
@@ -608,6 +609,21 @@ def preserved_envvars():
     ]
     return v + preserved_envvars_exported()
 
+def check_system_locale():
+    """Make sure the required system locale are available and configured"""
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
Signature · 1 line
2.20.1
#1 lore raw
Quoted 4 lines · Frank de Brabander via lists.openembedded.org

Get rid of the duplicate code and add extra check that the locale en_US.UTF-8 is available on the system. This new helper method is now located right above the method filter_environment() which sets LC_ALL environment variable to 'en_US.UTF-8'.

Nice:

foo@2b7837225ffb:~/b/test$ bitbake -p coreutils
Please make sure locale 'en_US.UTF-8' is available on your system

Steve,

It would be nice if you cherry-pick this back to langdale and kirkstone, it's a clean c-p.

../Randy

Quoted 77 lines

[YOCTO #10165]

---
  bin/bitbake        |  3 +--
  bin/bitbake-server |  5 +++--
  bin/bitbake-worker |  3 +--
  lib/bb/utils.py    | 16 ++++++++++++++++
  4 files changed, 21 insertions(+), 6 deletions(-)
bin/bitbake +1 -2
index 7cbf88f4..f869eb48 100755
@@ -25,8 +25,7 @@ except RuntimeError as exc:
  from bb import cookerdata
  from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
  __version__ = "2.2.0"
 
bin/bitbake-server +3 -2
index 825e9d56..454a3919 100755
@@ -12,8 +12,9 @@ warnings.simplefilter("default")
  import logging
  sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
  # Users shouldn't be running this code directly
  if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"): diff --git a/bin/bitbake-worker b/bin/bitbake-worker index 

3799b170..3cacdb0c 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -24,8 +24,7 @@ import subprocess from multiprocessing import Lock from threading import Thread -if sys.getfilesystemencoding() != "utf-8":

bin/bitbake-server +1 -1
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
  # Users shouldn't be running this code directly
  if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"): diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 

f4da3563..0df522b3 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py

bin/bitbake-server +1 -1
@@ -13,6 +13,7 @@ import errno import logging import bb import bb.msg 
+import locale import multiprocessing import fcntl import importlib @@ 
-608,6 +609,21 @@ def preserved_envvars(): ] return v + 

preserved_envvars_exported() +def check_system_locale(): + """Make sure the required system locale are available and configured"""

bin/bitbake-server +13 -1
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
  def filter_environment(good_vars):
      """
      Create a pristine environment for bitbake. This will remove variables that
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group.
View/Reply Online (#14139):https://lists.openembedded.org/g/bitbake-devel/message/14139 Mute This Topic:https://lists.openembedded.org/mt/95499228/3616765
Group Owner:[email protected] Unsubscribe:https://lists.openembedded.org/g/bitbake-devel/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

Signature · 2 lines
# Randy MacLeod
# Wind River Linux
#2 lore raw
Quoted 10 lines · Randy MacLeod

On 2022-12-06 13:18, Frank de Brabander via lists.openembedded.org wrote:

Get rid of the duplicate code and add extra check that the locale en_US.UTF-8 is available on the system. This new helper method is now located right above the method filter_environment() which sets LC_ALL environment variable to 'en_US.UTF-8'.

Nice:

Quoted 7 lines
foo@2b7837225ffb:~/b/test$ bitbake -p coreutils
Please make sure locale 'en_US.UTF-8' is available on your system

Steve,

It would be nice if you cherry-pick this back to langdale and kirkstone, it's a clean c-p.

OK, will do!

Steve

Quoted 102 lines

[YOCTO #10165]

---
 bin/bitbake        |  3 +--
 bin/bitbake-server |  5 +++--
 bin/bitbake-worker |  3 +--
 lib/bb/utils.py    | 16 ++++++++++++++++
 4 files changed, 21 insertions(+), 6 deletions(-)
bin/bitbake +1 -2
index 7cbf88f4..f869eb48 100755
@@ -25,8 +25,7 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 __version__ = "2.2.0"
 
bin/bitbake-server +3 -2
index 825e9d56..454a3919 100755
@@ -12,8 +12,9 @@ warnings.simplefilter("default")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"):
bin/bitbake-worker +1 -2
index 3799b170..3cacdb0c 100755
@@ -24,8 +24,7 @@ import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
lib/bb/utils.py +16 -1
index f4da3563..0df522b3 100644
@@ -13,6 +13,7 @@ import errno
 import logging
 import bb
 import bb.msg
+import locale
 import multiprocessing
 import fcntl
 import importlib
@@ -608,6 +609,21 @@ def preserved_envvars():
     ]
     return v + preserved_envvars_exported()
 
+def check_system_locale():
+    """Make sure the required system locale are available and configured"""
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
 
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group.
View/Reply Online (#14139): https://lists.openembedded.org/g/bitbake-devel/message/14139
Mute This Topic: https://lists.openembedded.org/mt/95499228/3616765
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

--
# Randy MacLeod
# Wind River Linux

#3 lore raw
Quoted 11 lines · Steve Sakoman
Quoted 9 lines · Randy MacLeod

On 2022-12-06 13:18, Frank de Brabander via lists.openembedded.org wrote:

Get rid of the duplicate code and add extra check that the locale en_US.UTF-8 is available on the system. This new helper method is now located right above the method filter_environment() which sets LC_ALL environment variable to 'en_US.UTF-8'.

Nice:

Quoted 8 lines
Quoted 7 lines
foo@2b7837225ffb:~/b/test$ bitbake -p coreutils
Please make sure locale 'en_US.UTF-8' is available on your system

Steve,

It would be nice if you cherry-pick this back to langdale and kirkstone, it's a clean c-p.

OK, will do!

Steve,

Did this cherry-pick slip through the cracks?

The master commit:

https://git.yoctoproject.org/poky/log/?qt=grep&q=Ensure+locale+en_US.UTF-8+is+available+on+the+system

and langdale

https://git.yoctoproject.org/poky/log/?h=langdale&qt=grep&q=Ensure+locale+en_US.UTF-8+is+available+on+the+system

but not on kirkstone

https://git.yoctoproject.org/poky/log/?h=kirkstone&qt=grep&q=Ensure+locale+en_US.UTF-8+is+available+on+the+system

Thanks,

../Randy

Quoted 79 lines

Steve

Quoted 76 lines

[YOCTO #10165]

---
  bin/bitbake        |  3 +--
  bin/bitbake-server |  5 +++--
  bin/bitbake-worker |  3 +--
  lib/bb/utils.py    | 16 ++++++++++++++++
  4 files changed, 21 insertions(+), 6 deletions(-)
bin/bitbake +1 -2
index 7cbf88f4..f869eb48 100755
@@ -25,8 +25,7 @@ except RuntimeError as exc:
  from bb import cookerdata
  from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
  __version__ = "2.2.0" diff --git a/bin/bitbake-server b/bin/bitbake-server index 
 825e9d56..454a3919 100755 --- a/bin/bitbake-server +++ b/bin/bitbake-server @@ -12,8 +12,9 @@ warnings.simplefilter("default")

import logging
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))

bin/bitbake +3 -2
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
  # Users shouldn't be running this code directly
  if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"): diff --git a/bin/bitbake-worker b/bin/bitbake-worker index 

3799b170..3cacdb0c 100755 --- a/bin/bitbake-worker +++ b/bin/bitbake-worker @@ -24,8 +24,7 @@ import subprocess from multiprocessing import Lock from threading import Thread -if sys.getfilesystemencoding() != "utf-8":

bin/bitbake +1 -1
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
  # Users shouldn't be running this code directly
  if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"): diff --git a/lib/bb/utils.py b/lib/bb/utils.py index 

f4da3563..0df522b3 100644 --- a/lib/bb/utils.py +++ b/lib/bb/utils.py

bin/bitbake +1 -0
@@ -13,6 +13,7 @@ import errno import logging import bb import bb.msg 
+import locale import multiprocessing import fcntl import importlib 
@@ -608,6 +609,21 @@ def preserved_envvars(): ] return v + 

preserved_envvars_exported() +def check_system_locale(): + """Make sure the required system locale are available and configured"""

bin/bitbake +13 -1
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
  def filter_environment(good_vars):
      """
      Create a pristine environment for bitbake. This will remove variables that
 
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group.
View/Reply Online (#14139):https://lists.openembedded.org/g/bitbake-devel/message/14139 Mute This Topic:https://lists.openembedded.org/mt/95499228/3616765
Group Owner:[email protected] Unsubscribe:https://lists.openembedded.org/g/bitbake-devel/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

--
# Randy MacLeod
# Wind River Linux

Signature · 2 lines
# Randy MacLeod
# Wind River Linux
#4 lore raw
Quoted 21 lines · Randy MacLeod

On 2023-03-08 16:41, Steve Sakoman wrote:

On Wed, Mar 8, 2023 at 11:08 AM Randy MacLeod
<[email protected]> wrote:

On 2022-12-06 13:18, Frank de Brabander via lists.openembedded.org wrote:

Get rid of the duplicate code and add extra check that the locale en_US.UTF-8 is available on the system. This new helper method is now located right above the method filter_environment() which sets LC_ALL environment variable to 'en_US.UTF-8'.

Nice:

foo@2b7837225ffb:~/b/test$ bitbake -p coreutils
Please make sure locale 'en_US.UTF-8' is available on your system

Steve,

Quoted 8 lines

It would be nice if you cherry-pick this back to langdale and kirkstone, it's a clean c-p.

OK, will do!

Steve,

Did this cherry-pick slip through the cracks?

Sadly, yes. Thanks for the reminder, I've got it in my test queue now.

Steve

Quoted 107 lines

[YOCTO #10165]

---
 bin/bitbake        |  3 +--
 bin/bitbake-server |  5 +++--
 bin/bitbake-worker |  3 +--
 lib/bb/utils.py    | 16 ++++++++++++++++
 4 files changed, 21 insertions(+), 6 deletions(-)
bin/bitbake +1 -2
index 7cbf88f4..f869eb48 100755
@@ -25,8 +25,7 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 __version__ = "2.2.0"
 
bin/bitbake-server +3 -2
index 825e9d56..454a3919 100755
@@ -12,8 +12,9 @@ warnings.simplefilter("default")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"):
bin/bitbake-worker +1 -2
index 3799b170..3cacdb0c 100755
@@ -24,8 +24,7 @@ import subprocess
 from multiprocessing import Lock
 from threading import Thread
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
lib/bb/utils.py +16 -1
index f4da3563..0df522b3 100644
@@ -13,6 +13,7 @@ import errno
 import logging
 import bb
 import bb.msg
+import locale
 import multiprocessing
 import fcntl
 import importlib
@@ -608,6 +609,21 @@ def preserved_envvars():
     ]
     return v + preserved_envvars_exported()
 
+def check_system_locale():
+    """Make sure the required system locale are available and configured"""
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
 
 
-=-=-=-=-=-=-=-=-=-=-=-

Links: You receive all messages sent to this group.
View/Reply Online (#14139): https://lists.openembedded.org/g/bitbake-devel/message/14139
Mute This Topic: https://lists.openembedded.org/mt/95499228/3616765
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-

--
# Randy MacLeod
# Wind River Linux

--
# Randy MacLeod
# Wind River Linux