Skip to contentSkip to conversation

build: Fix typo that breaks --fsanitize=leak check

PATCH Superseded Denis Kenzior 1 participant 6 messages Dec 13, 2023 Kbuild

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/6 build: Fix typo that breaks --fsanitize=leak check

---
 acinclude.m4 | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
acinclude.m4 +1 -2
index 4932ac6b..656888d7 100644
@@ -23,8 +23,7 @@ AC_DEFUN([AC_PROG_CC_ASAN], [
 ])
 
 AC_DEFUN([AC_PROG_CC_LSAN], [
-	AC_CACHE_CHECK([whether ${CC-cc} accepts -fsanitize=leak], ac_cv_prog_cc
-_lsan, [
+	AC_CACHE_CHECK([whether ${CC-cc} accepts -fsanitize=leak], ac_cv_prog_cc_lsan, [
 		echo 'void f(){}' > conftest.c
 		if test -z "`${CC-cc} -fsanitize=leak -c conftest.c 2>&1`"; then
 			ac_cv_prog_cc_lsan=yes
Signature · 1 line
2.43.0
#1 lore raw

PATCH 2/6 include: Allow multiple context types

---
 include/gprs-context.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
include/gprs-context.h +4 -4
index 75958284..81055d89 100644
@@ -33,10 +33,10 @@ struct ofono_modem;
 
 enum ofono_gprs_context_type {
 	OFONO_GPRS_CONTEXT_TYPE_ANY = 0,
-	OFONO_GPRS_CONTEXT_TYPE_INTERNET,
-	OFONO_GPRS_CONTEXT_TYPE_MMS,
-	OFONO_GPRS_CONTEXT_TYPE_WAP,
-	OFONO_GPRS_CONTEXT_TYPE_IMS,
+	OFONO_GPRS_CONTEXT_TYPE_INTERNET	= 0x0001,
+	OFONO_GPRS_CONTEXT_TYPE_MMS		= 0x0002,
+	OFONO_GPRS_CONTEXT_TYPE_WAP		= 0x0004,
+	OFONO_GPRS_CONTEXT_TYPE_IMS		= 0x0008,
 };
 
 struct ofono_gprs_primary_context {
Signature · 1 line
2.43.0
#2 lore raw

PATCH 3/6 doc: docs for intermediate provisioning db format

In reply to Denis Kenzior

Introduce a new provisioning file format to be used by oFono. This format will be an 'intermediate' format, that will be converted to a binary format for use on the actual device. This allows expensive and error prone (not to mention hard to secure) parsing of XML files (such asmobile-broadband-provider-info or Android apns-conf.xml) to be avoided.

JSON was chosen since it is a much more readable format compared to XML.

---
 doc/provision.rst | 139 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 doc/provision.rst
doc/provision.rst new file +139 -0
new file mode 100644
index 00000000..693b38a3
@@ -0,0 +1,139 @@
+Carrier Settings Provisioning Database
+======================================
+
+Carrier settings provisioning is performed automatically whenever a new,
+not previously used SIM card is detected for the first time.  Settings such as
+APN, username, password settings as well as the default bearer settings can be
+provisioned automatically if a matching database entry is found.
+
+Matches are performed based on the following information:
+    * SIM Mobile Country Code (MCC)
+    * SIM Mobile Network Code (MNC)
+    * SIM Service Provider Name (SPN)
+
+The carrier settings provisioning database is represented in JSON format and is
+converted to a binary format at installation time.
+
+JSON Structure
+--------------
+**(List):**
+    List of mobile network provider objects.  Top level element.
+
+    Example:
+    ``[{"name": "Operator XYZ", ...}, {"name": "Operator ZYX", ...}]``
+
+    - **name (String):**
+        The name of the mobile network provider.  This field is a freeform
+        string that identifies the provider.  This field is used purely for
+        human consumption and not used by the provisioning logic.
+
+        This field is `required`.
+
+        Example: "Operator XYZ"
+
+    - **ids (List of Strings):**
+        Unique identifiers associated with the mobile network provider.  This
+        is a list of all MCC+MNC identifiers associated with this provider.
+
+        This field is `required`.
+
+        Example: ``["99955", "99956", "99901", "99902"]``
+
+    - **spn (String):**
+        Service Provider Name associated with the mobile network provider.  This
+        field is typically used to differentiate MVNOs from non-MVNO providers.
+
+        This field is `optional`.
+
+        Example: "ZYX"
+
+    - **apns (List):**
+        List of access points associated with the mobile network provider.  At
+        least one entry must be present in the list.
+
+        This field is `required`.
+
+        - **name (String):**
+            The descriptive name of the access point.  If present, will be
+            reflected in the oFono context name after successful provisioning.
+
+            This field is `optional`.
+
+            Example: "Internet"
+
+        - **apn (String):**
+            Access Point Name - Setting required for successful bearer
+            activation.  Provided by the carrier.
+
+            This field is `required`.
+
+            Example: "internet"
+
+        - **type (List of Strings):**
+            The types of connections supported by the access point.  The
+            following types are recognized:
+
+            - "internet": Used for general internet access.
+            - "mms": Used for Multimedia Messaging Service (MMS).
+            - "wap": Used for Wireless Application Protocol (WAP).
+            - "ims": Used for IP Multimedia Subsystem (IMS).
+            - "supl": Used for Secure User Plane Location (SUPL).
+            - "ia": Used for Initial Attach in LTE networks.
+
+            This field is `required`.
+
+            Example: ``["internet", "mms"]``
+
+        - **authentication (String):**
+            Authentication method used for the connection.  The following types
+            are recognized:
+
+            - "chap": CHAP authentication
+            - "pap": PAP authentication
+            - "none": No authentication is used
+
+            This field is `optional`.
+
+            Example: "none"
+
+        - **username (String):**
+            Username used for autenticating to the access point.
+
+            This field is `optional`.
+
+            Example: "username"
+
+        - **password (String):**
+            Password used for autenticating to the access point.
+
+            This field is `optional`.
+
+            Example: "temp123"
+
+        - **protocol (String):**
+            Network protocol used for the connection.  The following types are
+            recognized:
+
+            - "ipv4": IPv4 only
+            - "ipv6": IPv6 only
+            - "ipv4v6": Dual protocol, both IPv4 and IPv6 will be negotiated
+
+            If omitted, then `"ipv4v6"` will be assumed.
+
+            This field is `optional`.
+
+            Example: "ipv4"
+
+        - **mmsc (String):**
+            Multimedia Messaging Service Center - URL for MMS.
+
+            This field is `required` for MMS contexts.
+
+            Example: "foobar.mmsc:80"
+
+        - **mmsproxy (String):**
+            Proxy server for Multimedia Messaging Service (MMS).
+
+            This field is `optional`.
+
+            Example: "mms.proxy.net"
Signature · 1 line
2.43.0
#3 lore raw

PATCH 4/6 tools: Add provision.py

In reply to Denis Kenzior

Introduce a new tool that will convert the intermediate JSON format (documented in doc/provision.rst) to the binary provisioning format, which will be used by ofonod's default provisioning plugin for automatically setting up carrier specific settings.

The tool also supports import of and conversion of 'mobile-broadband-provider-info' XML files to the new intermediate JSON file format. This is accomplished using:
% tools/provision.py mbpi-convert --outfile=provision.json

Conversion of JSON intermediate format to binary format is accomplished using:
% tools/provision.py generate --infile=provision.json

By default, the output will be placed in the same directory in the file 'provision.db'. Alternatively, the output file can be specified using the --outfile option.

Finally, the tool supports a simple selftest method, which can be invoked as follows:
% tools/provision.py selftest

---
 tools/provision.py | 727 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 727 insertions(+)
 create mode 100755 tools/provision.py
tools/provision.py new file +727 -0
new file mode 100755
index 00000000..79273277
@@ -0,0 +1,727 @@
+#!/usr/bin/python3
+#
+# oFono - Open Source Telephony
+# Copyright (C) 2023  Cruise, LLC
+#
+# SPDX-License-Identifier: GPL-2.0-only
+import xml.etree.ElementTree as ET
+import sys
+import json
+import bisect
+from argparse import ArgumentParser, FileType
+from pathlib import Path
+import random
+import struct
+import ctypes
+
+class ProviderInfo:
+    sort_order_map = { v : pos for pos, v in
+                      enumerate( ['name',
+                                  'apn',
+                                  'type',
+                                  'protocol',
+                                  'mmsc',
+                                  'mmsproxy',
+                                  'authentication',
+                                  'username',
+                                  'password'] ) }
+
+    @classmethod
+    def rawimport(cls, entry):
+        if 'name' not in entry:
+            raise SystemExit('No name for entry: ' + str(entry))
+
+        info = ProviderInfo(entry['name'])
+
+        for networkid in entry.get('ids', []):
+            if not info.add_id(networkid):
+                raise SystemExit('Invalid network id: ' + str(networkid))
+
+        if 'spn' in entry:
+            if not info.set_spn(entry['spn']):
+                raise SystemExit('Invalid spn: ' + str(spn))
+
+        for apn in entry.get('apns', []):
+            if not info.add_context(apn):
+                raise SystemExit('Invalid apn: ' + str(apn))
+
+        if not info.is_valid():
+            raise SystemExit('Invalid entry: ' + str(entry))
+
+        return info
+
+    def __init__(self, name):
+        self.context_list = []
+        self.mccmnc_list = []
+        self.name = name
+        self.spn = None
+
+    @staticmethod
+    def is_valid_id(id_string, expected_lengths):
+        """
+        Check if the identifier string is valid.
+
+        Parameters:
+        - id_string: The id string to check.
+        - expected_lengths (tuple): A tuple representing the valid range of
+          lengths.
+
+        Returns:
+        - bool: True if the MCC string is valid, False otherwise.
+        """
+        if not id_string.isdigit():
+            return False
+
+        if len(id_string) not in expected_lengths:
+            return False
+
+        if int(id_string) == 0:
+            return False
+
+        return True
+
+    def add_mccmnc(self, mcc, mnc):
+        if not self.is_valid_id(mcc, (3,)) or not self.is_valid_id(mnc, (2, 3)):
+            return False
+
+        bisect.insort(self.mccmnc_list, mcc + mnc)
+        return True
+
+    def add_id(self, mccmnc):
+        if not self.is_valid_id(mccmnc, (5,6)):
+            return False
+
+        if int(mccmnc[:3]) == 0 or int(mccmnc[3:]) == 0:
+            return False
+
+        bisect.insort(self.mccmnc_list, mccmnc)
+        return True
+
+    def set_spn(self, spn):
+        if len(spn) == 0 or len(spn) > 254:
+            return False
+
+        self.spn = spn
+        return True
+
+    def add_context(self, info):
+        info = dict(sorted(info.items(),
+                      key = lambda pair: self.sort_order_map[pair[0]]))
+        self.context_list.append(info)
+
+        return True
+
+    def is_valid(self):
+        return len(self.context_list) and len(self.mccmnc_list)
+
+    def __str__(self):
+        s = 'Provider \'' + self.name + '\''
+
+        if (self.spn != None):
+            s += ' [SPN:\'' + self.spn + '\']'
+
+        s+= ' ' + str(self.mccmnc_list) + '\n'
+
+        for context in self.context_list:
+            s += '\t' + str(context) + '\n'
+
+        return s
+
+class MobileBroadbandProviderInfo:
+    usage_to_type = { 'internet' : ['internet'],
+                        'mms' : ['mms'],
+                        'wap' : ['wap'],
+                        'mms-internet-hipri' : ['internet', 'mms'],
+                        'mms-internet-hipri-fota' : ['internet','mms'],
+                     }
+    @classmethod
+    def type_from_usage(cls, usage):
+        return cls.usage_to_type[usage]
+
+    def __init__(self, xml_path):
+        self.tree = ET.parse(xml_path)
+
+    def parse(self, xml_path):
+        providers = []
+
+        try:
+            tree = ET.parse(xml_path)
+            root = tree.getroot()
+
+            for provider in root.findall('.//provider'):
+                name = provider.find('name')
+                if name is None or not name.text:
+                    continue;
+
+                info = ProviderInfo(name.text)
+
+                for networkid in provider.findall('gsm/network-id'):
+                    info.add_mccmnc(networkid.get('mcc'), networkid.get('mnc'))
+
+                for apn in provider.findall('gsm/apn'):
+                    context = {}
+
+                    context['apn'] = apn.get('value')
+                    if context['apn'] == None:
+                        continue
+
+                    # Usage is missing for some APNs, skip such contexts for now
+                    usage = apn.find('usage')
+                    if usage is None or usage.get('type') is None:
+                        continue;
+
+                    context['type'] = self.type_from_usage(usage.get('type'))
+                    if context['type'] == None:
+                        sys.stderr.write("Unable to convert type: %s\n" %
+                                            usage.get('type'))
+                        continue
+
+                    if 'mms' in context['type']:
+                        mmsc = apn.find('mmsc')
+
+                        # Ignore MMS contexts with no MMSC since it is needed
+                        # to send messages
+                        if mmsc is None or not mmsc.text:
+                            continue
+
+                        context['mmsc'] = mmsc.text
+
+                        mmsproxy = apn.find('mmsproxy')
+                        if mmsproxy is not None and mmsproxy.text:
+                            context['mmsproxy'] = mmsproxy.text
+
+                    username = apn.find('username')
+                    if username is not None and username.text:
+                        context['username'] = username.text
+
+                    password = apn.find('password')
+                    if password is not None and password.text:
+                        context['password'] = password.text
+
+                    authentication = apn.find('authentication')
+                    if authentication is not None:
+                        context['authentication'] = authentication.get('method')
+
+                    context_name = apn.find('name')
+                    if context_name != None:
+                        context['name'] = context_name.text
+
+                    info.add_context(context)
+
+                if info.is_valid():
+                    providers.append(info)
+
+        except ET.ParseError as e:
+            print(f"Error parsing XML: {e}")
+
+        return providers
+
+class ProvisionContext(ctypes.LittleEndianStructure):
+    _pack_ = 1
+    _fields_ = [
+        ('type', ctypes.c_uint32),
+        ('protocol', ctypes.c_uint32),
+        ('authentication', ctypes.c_uint32),
+        ('reserved', ctypes.c_uint32),
+        ('name_offset', ctypes.c_uint64),
+        ('apn_offset', ctypes.c_uint64),
+        ('username_offset', ctypes.c_uint64),
+        ('password_offset', ctypes.c_uint64),
+        ('mmsproxy_offset', ctypes.c_uint64),
+        ('mmsc_offset', ctypes.c_uint64)
+    ]
+
+    authentication_dict = { 'chap' : 0, 'pap' : 1, 'none' : 2 }
+    protocol_dict = { 'ipv4' : 0, 'ipv6' : 1, 'ipv4v6' : 2 }
+    attrs = ['name', 'apn', 'username', 'password', 'mmsproxy', 'mmsc']
+
+    @classmethod
+    def type_to_context_type(cls, types):
+        r = 0
+
+        for t in types:
+            if t == 'internet':
+                r |= 0x0001
+            elif t == 'mms':
+                r |= 0x0002
+            elif t == 'wap':
+                r |= 0x0004
+            elif t == 'ims':
+                r |= 0x0008
+            elif t == 'supl':
+                r |= 0x0010
+            elif t == 'ia':
+                r |= 0x0020
+
+        return r
+
+    def __init__(self, apn, strings):
+        self.type = self.type_to_context_type(apn['type'])
+        self.protocol = self.protocol_dict[apn.get('protocol', 'ipv4v6')]
+        self.authentication = self.authentication_dict[apn.get('authentication',
+                                                               'chap')]
+
+        for s in self.attrs:
+            offset = strings.add_string(apn.get(s, None))
+            setattr(self, s + '_offset', offset)
+
+class ProvisionData(ctypes.LittleEndianStructure):
+    _pack_ = 1
+    _fields_ = [
+        ('spn_offset', ctypes.c_uint64),
+        ('context_offset', ctypes.c_uint64)
+    ]
+
+    def __init__(self, spn, offset, strings):
+        self.spn_offset = strings.add_string(spn)
+        self.context_offset = offset
+
+class ProvisionNode(ctypes.LittleEndianStructure):
+    _pack_ = 1
+    _fields_ = [
+        ('bit_offsets', ctypes.c_uint64 * 2),
+        ('mccmnc', ctypes.c_uint32),
+        ('diff', ctypes.c_int32),
+        ('provision_data_count', ctypes.c_uint64)
+    ]
+
+    style = "bold"
+    fmt_connection = '\t"%s/%d" -> "%s/%d"[color="#%06x"];\n'
+    fmt_declaration = '\t"%s/%d"[style=%s, color="#%06x"];\n'
+    red = 0xff0000
+    green = 0x00ff00
+
+    def __init__(self, key, diff):
+        self.bit = [None, None]
+        self.key = key
+        self.diff = diff
+        self.entries = {}
+        self.node_offset = 0
+
+    def choose(self, key):
+        return (key >> (31 - self.diff)) & 1
+
+    def print_graphviz(self, f):
+        f.write(self.fmt_declaration % (format(self.key, '032b'),
+                                        self.diff, self.style,
+                                        random.randint(0, 0x00ffffff)))
+        f.write(self.fmt_connection % (format(self.key, '032b'), self.diff,
+                                       format(self.bit[0].key, '032b'),
+                                       self.bit[0].diff, self.red))
+        f.write(self.fmt_connection % (format(self.key, '032b'), self.diff,
+                                       format(self.bit[1].key, '032b'),
+                                       self.bit[1].diff, self.green))
+
+        if (self.diff < self.bit[0].diff):
+            self.bit[0].print_graphviz(f)
+
+        if (self.diff < self.bit[1].diff):
+            self.bit[1].print_graphviz(f)
+
+    def __str__(self):
+        s = format(self.key, '032b') + '/' + str(self.diff)
+        return s
+
+class MccMncTree:
+    @staticmethod
+    def clz(v):
+        count = 32
+        while count and v:
+            v = v >> 1
+            count = count - 1
+
+        return count
+
+    @staticmethod
+    def diff(key1, key2):
+        xor = key1 ^ key2;
+        return MccMncTree.clz(xor)
+
+    def __init__(self):
+        self.root = ProvisionNode(key = 0, diff = -1)
+        self.root.bit[0] = self.root
+        self.root.bit[1] = self.root
+        self.n_nodes = 1
+
+    def print_graphviz(self):
+        f = open("step%d.dot" % self.n_nodes, "w")
+        # Use 'dot -Tx11' to visualize
+        f.write('digraph trie {\n')
+        self.root.print_graphviz(f)
+        f.write('}\n')
+        f.close()
+
+    def find_closest(self, key):
+        parent = self.root
+        child = self.root.bit[0]
+
+        while parent.diff < child.diff:
+            parent = child
+            child = child.bit[child.choose(key)]
+
+        return child
+
+    def find(self, key):
+        found = self.find_closest(key)
+        if found.key == key:
+            return found
+
+        return None
+
+    def insert(self, key, attr, value):
+        node = self.find_closest(key);
+        if node.key == key:
+            node.entries[attr] = value
+            return
+
+        bit = self.diff(node.key, key)
+        parent = self.root
+        child = self.root.bit[0]
+
+        while (parent.diff < child.diff) and (child.diff < bit):
+            parent = child
+            child = child.bit[child.choose(key)]
+
+        node = ProvisionNode(key, bit)
+        bit = node.choose(key)
+        node.bit[bit] = node
+        node.bit[not bit] = child
+
+        node.entries[attr] = value
+
+        if parent == self.root:
+            self.root.bit[0] = node
+        else:
+            bit = parent.choose(key)
+            parent.bit[bit] = node
+
+        self.n_nodes += 1
+
+    def traverse_recursive(self, node, bit, visitor):
+        if node == self.root:
+            return
+
+        if node.diff <= bit:
+            visitor.visit(node)
+            return
+
+        self.traverse_recursive(node.bit[0], node.diff, visitor)
+        self.traverse_recursive(node.bit[1], node.diff, visitor)
+
+    def traverse(self, visitor):
+        self.traverse_recursive(self.root.bit[0], -1, visitor)
+
+class StringAccumulator:
+    def __init__(self):
+        self.data = bytearray(b'\x00') # So offsets are never 0 used for NULL
+        self.offsets = {}
+
+    def add_string(self, s):
+        if s is None:
+            return 0
+
+        if s in self.offsets:
+            return self.offsets[s]
+
+        offset = len(self.data)
+        self.data.extend(s.encode('utf-8'))
+        self.data.append(0)
+        self.offsets[s] = offset
+
+        return offset
+
+    def get_bytes(self):
+        return self.data
+
+class ProvisionDatabase(ctypes.LittleEndianStructure):
+    _pack_ = 1
+    _fields_ = [
+        ('version', ctypes.c_uint64),
+        ('file_size', ctypes.c_uint64),
+        ('header_size', ctypes.c_uint64),
+        ('node_struct_size', ctypes.c_uint64),
+        ('provision_data_struct_size', ctypes.c_uint64),
+        ('context_struct_size', ctypes.c_uint64),
+        ('nodes_offset', ctypes.c_uint64),
+        ('nodes_size', ctypes.c_uint64),
+        ('contexts_offset', ctypes.c_uint64),
+        ('contexts_size', ctypes.c_uint64),
+        ('strings_offset', ctypes.c_uint64),
+        ('strings_size', ctypes.c_uint64)
+    ]
+
+    class CalculateNodeOffsetVisitor:
+        def __init__(self):
+            self.current_offset = 0
+        def visit(self, node):
+            node.node_offset = self.current_offset
+
+            # Node data is followed by at least one ProvisionData object, with
+            # the only exception being root, which has no data by definition
+            self.current_offset += ctypes.sizeof(ProvisionNode)
+            self.current_offset += (ctypes.sizeof(ProvisionData) *
+                                    len(node.entries))
+
+    class SerializeVisitor:
+        def __init__(self, buffer):
+            self.buffer = buffer
+
+        def visit(self, node):
+            # Node doesn't quite fit the C structure definition, so do this
+            # manually by using struct.pack
+            self.buffer.extend(struct.pack('<QQIiQ',
+                                           node.bit[0].node_offset,
+                                           node.bit[1].node_offset,
+                                           node.key, node.diff,
+                                           len(node.entries)))
+
+            for spn in sorted(node.entries):
+                pd = node.entries[spn]
+                self.buffer.extend(bytes(pd))
+
+    def __init__(self, provider_infos):
+        self.strings = StringAccumulator()
+        self.contexts = bytearray()
+        self.tree = MccMncTree()
+
+        for info in provider_infos:
+            pd = ProvisionData(info.spn, len(self.contexts), self.strings)
+
+            self.contexts.extend(struct.pack('<Q', len(info.context_list)))
+
+            for context in info.context_list:
+                self.contexts.extend(bytes(ProvisionContext(context,
+                                                            self.strings)))
+
+            for mccmnc in info.mccmnc_list:
+                # Sort None spns as '' so they're first in the list when
+                # the SerializeVisitor sorts the entries dict
+                spn = info.spn if info.spn is not None else ''
+
+                # 2 and 3 byte MNCs are treated differently, even if evaluate
+                # to the same integer.  For example, 02 and 002 are different
+                # MNCs.  In practice this doesn't actually happen except on
+                # test networks, but account for this possibility by using the
+                # upper 10 bits for the MCC, followed by a single bit which
+                # signifies whether a 3 byte MNC is used, followed by 10 bits
+                # of the MNC
+                key = int(mccmnc[:3]) << 11 | int(mccmnc[3:])
+                if len(mccmnc[3:]) == 3:
+                    key |= 1 << 10
+
+                self.tree.insert(key, spn, pd)
+
+        visitor = self.CalculateNodeOffsetVisitor()
+        visitor.visit(self.tree.root)
+        self.tree.traverse(visitor)
+
+        self.version = 1
+        self.header_size = ctypes.sizeof(ProvisionDatabase)
+        self.file_size = self.header_size
+        self.node_struct_size = ctypes.sizeof(ProvisionNode)
+        self.provision_data_struct_size = ctypes.sizeof(ProvisionData)
+        self.context_struct_size = ctypes.sizeof(ProvisionContext)
+        self.nodes_offset = self.header_size
+        self.nodes_size = visitor.current_offset
+        self.file_size += self.nodes_size
+        self.contexts_offset = self.nodes_offset + self.nodes_size
+        self.contexts_size = len(self.contexts)
+        self.file_size += self.contexts_size
+        self.strings_offset = self.contexts_offset + self.contexts_size
+        self.strings_size = len(self.strings.get_bytes())
+        self.file_size += self.strings_size
+
+    def serialize(self):
+        buffer = bytearray()
+        buffer.extend(bytes(self))
+
+        visitor = self.SerializeVisitor(buffer)
+        visitor.visit(self.tree.root)
+        self.tree.traverse(visitor)
+
+        buffer.extend(self.contexts)
+        buffer.extend(self.strings.get_bytes())
+
+        return buffer
+
+class ProviderInfoJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, ProviderInfo):
+            asdict = { 'name' : obj.name, 'ids' : obj.mccmnc_list }
+
+            if (obj.spn != None):
+                asdict['spn'] = obj.spn
+
+            asdict.update({'apns' : obj.context_list})
+            return asdict
+
+        return json.JSONEncoder.default(self, obj)
+
+def mbpi_convert(args):
+    xml_path = '/usr/share/mobile-broadband-provider-info/serviceproviders.xml'
+    mbpi = MobileBroadbandProviderInfo(xml_path)
+    provider_infos = mbpi.parse(xml_path)
+
+    try:
+        if args.outfile is None:
+            out = sys.stdout
+        else:
+            out = args.outfile.open('w', encoding='utf-8')
+
+        with out as outfile:
+            json.dump(provider_infos, outfile, ensure_ascii=False, indent=2,
+                      cls=ProviderInfoJSONEncoder)
+    except ValueError as e:
+        raise SystemExit(e)
+
+def generate(args):
+    try:
+        json_dict = json.load(args.infile)
+    except ValueError as e:
+        raise SystemExit(e)
+
+    provider_infos = []
+
+    for entry in json_dict:
+        info = ProviderInfo.rawimport(entry)
+        provider_infos.append(info)
+
+    db = ProvisionDatabase(provider_infos)
+    with args.outfile.open('wb') as outfile:
+        outfile.write(db.serialize())
+
+def selftest(args):
+    tree = MccMncTree()
+
+    # Generate random key this many times and insert it into the tree
+    # There will be some collisions, so count the number of items inserted
+    # Then run the lookup and make sure the same number of items can be found
+    times = 100000
+    for i in range(0, times):
+        key = random.randint(10000, 999999)
+        tree.insert(key, None, None)
+
+    n_inserted = tree.n_nodes
+    print("Created a tree with %d nodes (1 root)" % n_inserted)
+
+    n_found = 0
+    expected_keys = []
+
+    for i in range(10000, 1000000):
+        if tree.find(i):
+            n_found += 1
+            expected_keys.append(i)
+
+    expected_keys = sorted(expected_keys)
+
+    print("Found %d nodes (not including root)" % n_found)
+    assert n_found == n_inserted - 1
+
+    class GetKeysVisitor:
+        def __init__(self):
+            self.keys = []
+
+        def visit(self, node):
+            self.keys.append(node.key)
+
+    visitor = GetKeysVisitor()
+    tree.traverse(visitor)
+    assert visitor.keys == expected_keys
+
+    sample_json = """[
+    {
+      "name": "Operator XYZ",
+      "ids": [
+        "99955", "99956", "99901", "99902"
+      ],
+      "apns": [
+        {
+          "name": "Internet",
+          "apn": "internet",
+          "type": [
+            "internet"
+          ],
+          "authentication": "none",
+          "protocol": "ipv4"
+        },
+        {
+          "name": "IMS+MMS",
+          "apn": "imsmms",
+          "type": [
+            "ims", "mms"
+          ],
+          "mmsc": "foobar.mmsc:80",
+          "mmsproxy": "mms.proxy.net",
+          "authentication": "pap",
+          "protocol": "ipv6"
+        }
+      ]
+    },
+    {
+      "name": "Operator ZYX",
+      "ids": [
+        "99998", "99999", "99901", "99902"
+      ],
+      "spn": "ZYX",
+      "apns": [
+        {
+          "name": "ZYX",
+          "apn": "zyx",
+          "type": [
+            "internet"
+          ],
+          "authentication": "none",
+          "protocol": "ipv4"
+        }
+      ]
+    }
+    ]"""
+    try:
+        json_dict = json.loads(sample_json)
+    except ValueError as e:
+        raise SystemExit(e)
+
+    provider_infos = []
+
+    for entry in json_dict:
+        info = ProviderInfo.rawimport(entry)
+        provider_infos.append(info)
+
+    db = ProvisionDatabase(provider_infos)
+    expected_strings = bytearray(b'\x00Internet\x00internet\x00IMS+MMS\x00' +
+                b'imsmms\x00mms.proxy.net\x00foobar.mmsc:80\x00ZYX\x00zyx\x00')
+    assert(expected_strings == db.strings.get_bytes())
+
+    with open('sample.db', 'wb') as outfile:
+        outfile.write(db.serialize())
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description='Parse command line arguments')
+    subparsers = parser.add_subparsers(title='subcommands', dest='command')
+
+    # mbpi-convert command
+    mbpi_convert_parser = subparsers.add_parser('mbpi-convert',
+                        help='Convert mobile-broadband-provider-info database')
+    mbpi_convert_parser.add_argument('--outfile', type=Path, default=None,
+                        help='Output file path', required=False,)
+    mbpi_convert_parser.set_defaults(func=mbpi_convert)
+
+    # generate command
+    generate_parser = subparsers.add_parser('generate',
+                                            help='Generate binary provider db')
+    generate_parser.add_argument('--outfile', type=Path, default='provision.db',
+                                 help='Output file path', required=False)
+    generate_parser.add_argument('--infile', type=FileType(encoding='utf-8'),
+                                 help='Input JSON db', default=sys.stdin)
+    generate_parser.set_defaults(func=generate)
+
+    # selftest command
+    selftest_parser = subparsers.add_parser('selftest',
+                        help='Run self-tests')
+    selftest_parser.set_defaults(func=selftest)
+
+    args = parser.parse_args()
+    if hasattr(args, 'func'):
+        args.func(args)
+    else:
+        parser.print_help()
Signature · 1 line
2.43.0
#4 lore raw

PATCH 5/6 plugins: Support the new binary provisioning format

In reply to Denis Kenzior

Binary provisioning format is based on a patricia trie / crit-bit tree, with each MCC + MNC combination represented by a single node. Each node contains an array of { SPN, offset } pairs, with the offset pointing to the region of memory where the context information resides.

Each node also has two offsets, corresponding to left and right children of the node, as well as the position of the critical bit, based on which the tree is traversed.

All strings are placed in the last section and are nil terminated. Structures are designed with 8-byte alignment and stored in little endian format, which will require no conversion for the vast majority of platforms in use.

---
 Makefile.am           |   2 +
 plugins/provisiondb.c | 436 ++++++++++++++++++++++++++++++++++++++++++
 plugins/provisiondb.h |  18 ++
 3 files changed, 456 insertions(+)
 create mode 100644 plugins/provisiondb.c
 create mode 100644 plugins/provisiondb.h
Makefile.am +2 -0
index e7fd030f..11106162 100644
@@ -656,6 +656,8 @@ endif
 if PROVISION
 builtin_sources += plugins/mbpi.h plugins/mbpi.c
 
+builtin_sources += plugins/provisiondb.h plugins/provisiondb.c
+
 builtin_modules += provision
 builtin_sources += plugins/provision.c
 
plugins/provisiondb.c new file +436 -0

436 lines not on this page · raw message

plugins/provisiondb.h new file +18 -0
new file mode 100644
index 00000000..19a738fd
@@ -0,0 +1,18 @@
+/*
+ *  oFono - Open Source Telephony
+ *  Copyright (C) 2023  Cruise, LLC
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+struct ofono_gprs_provision_data;
+struct provision_db;
+
+struct provision_db *provision_db_new(const char *pathname);
+struct provision_db *provision_db_new_default(void);
+void provision_db_free(struct provision_db *pdb);
+
+int provision_db_lookup(struct provision_db *pdb,
+			const char *mcc, const char *mnc, const char *spn,
+			struct ofono_gprs_provision_data **items,
+			size_t *n_items);
Signature · 1 line
2.43.0
#5 lore raw

PATCH 6/6 tools: lookup-apn: Use the new provision_db utils

In reply to Denis Kenzior

Migrate lookup-apn tool to use the new provision.db binary file format.

The typical usage is now changed as follows:
% tools/lookup-apn --file=sample.db 999 02

---
 Makefile.am        |   6 +-
 tools/lookup-apn.c | 173 ++++++++++++++++++++++++++-------------------
 2 files changed, 103 insertions(+), 76 deletions(-)
Makefile.am +3 -3
index 11106162..a61cc9f4 100644
@@ -663,7 +663,6 @@ builtin_sources += plugins/provision.c
 
 builtin_modules += file_provision
 builtin_sources += plugins/file-provision.c
-
 endif
 
 if MAINTAINER_MODE
@@ -1010,8 +1009,9 @@ tools_auto_enable_LDADD = gdbus/libgdbus-internal.la @GLIB_LIBS@ @DBUS_LIBS@
 tools_get_location_SOURCES = tools/get-location.c
 tools_get_location_LDADD = @GLIB_LIBS@ @DBUS_LIBS@
 
-tools_lookup_apn_SOURCES = plugins/mbpi.c plugins/mbpi.h tools/lookup-apn.c
-tools_lookup_apn_LDADD = @GLIB_LIBS@
+tools_lookup_apn_SOURCES = plugins/provisiondb.h plugins/provisiondb.c \
+				tools/lookup-apn.c
+tools_lookup_apn_LDADD = $(ell_ldadd)
 
 tools_tty_redirector_SOURCES = tools/tty-redirector.c
 tools_tty_redirector_LDADD = @GLIB_LIBS@
tools/lookup-apn.c +100 -73

215 lines not on this page · raw message

Signature · 1 line
2.43.0