sunnypilot v2026.002.000

version: sunnypilot v2026.002.000 (staging)
date: 2026-07-21T07:44:02
master commit: 2c334ede443d7391d27575af8c854a095ba702a8

openpilot/common/api/__init__.py

@@ -0,0 +1,26 @@
+from openpilot.common.api.comma_connect import CommaConnectApi
+
+
+class Api:
+  def __init__(self, dongle_id):
+    self.service = CommaConnectApi(dongle_id)
+
+  def request(self, method, endpoint, **params):
+    return self.service.request(method, endpoint, **params)
+
+  def get(self, *args, **kwargs):
+    return self.service.get(*args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.service.post(*args, **kwargs)
+
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self.service.get_token(payload_extra, expiry_hours)
+
+
+def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
+  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+
+
+def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+  return CommaConnectApi(None).get_key_pair()

openpilot/common/api/base.py

@@ -0,0 +1,72 @@
+import jwt
+import os
+import requests
+import unicodedata
+from datetime import datetime, timedelta, UTC
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.version import get_version
+
+# name: jwt signature algorithm
+KEYS = {"id_rsa": "RS256",
+        "id_ecdsa": "ES256"}
+
+
+class BaseApi:
+  def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
+    self.dongle_id = dongle_id
+    self.api_host = api_host
+    self.user_agent = user_agent
+    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours),
+      **extra_payload
+    }
+    if payload_extra is not None:
+      payload.update(payload_extra)
+    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self._get_token(payload_extra, expiry_hours)
+
+  def remove_non_ascii_chars(self, text):
+    normalized_text = unicodedata.normalize('NFD', text)
+    ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
+    return ascii_encoded_text.decode()
+
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+    headers = {}
+    if access_token is not None:
+      headers['Authorization'] = "JWT " + access_token
+
+    version = self.remove_non_ascii_chars(get_version())
+    headers['User-Agent'] = self.user_agent + version
+
+    # TODO: add session to Api
+    req = requests if session is None else session
+    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
+
+  @staticmethod
+  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+    for key in KEYS:
+      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
+        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
+          return KEYS[key], private.read(), public.read()
+    return None, None, None

openpilot/common/api/comma_connect.py

@@ -0,0 +1,11 @@
+import os
+
+from openpilot.common.api.base import BaseApi
+
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+
+class CommaConnectApi(BaseApi):
+  def __init__(self, dongle_id):
+    super().__init__(dongle_id, API_HOST)
+    self.user_agent = "openpilot-"

openpilot/common/basedir.py

@@ -0,0 +1,4 @@
+import os
+
+
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))

openpilot/common/constants.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+# conversions
+class CV:
+  # Speed
+  MPH_TO_KPH = 1.609344
+  KPH_TO_MPH = 1. / MPH_TO_KPH
+  MS_TO_KPH = 3.6
+  KPH_TO_MS = 1. / MS_TO_KPH
+  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
+  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
+  MS_TO_KNOTS = 1.9438
+  KNOTS_TO_MS = 1. / MS_TO_KNOTS
+
+  # Angle
+  DEG_TO_RAD = np.pi / 180.
+  RAD_TO_DEG = 1. / DEG_TO_RAD
+
+  # Mass
+  LB_TO_KG = 0.453592
+
+
+ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

openpilot/common/esim/__init__.py

@@ -0,0 +1,3 @@
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+
+__all__ = ["LPABase", "LPAError", "LPAProfileNotFoundError", "Profile"]

openpilot/common/esim/base.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+class LPAError(RuntimeError):
+  pass
+
+
+class LPAProfileNotFoundError(LPAError):
+  pass
+
+
+@dataclass
+class Profile:
+  iccid: str
+  nickname: str
+  enabled: bool
+  provider: str
+
+  @property
+  def is_comma(self) -> bool:
+    return self.provider == 'Webbing' and self.iccid.startswith('8985235')
+
+
+class LPABase(ABC):
+  @abstractmethod
+  def list_profiles(self) -> list[Profile]:
+    pass
+
+  @abstractmethod
+  def get_active_profile(self) -> Profile | None:
+    pass
+
+  @abstractmethod
+  def delete_profile(self, iccid: str) -> None:
+    pass
+
+  @abstractmethod
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    pass
+
+  @abstractmethod
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    pass
+
+  @abstractmethod
+  def switch_profile(self, iccid: str) -> None:
+    pass
+
+  def process_notifications(self) -> None:
+    pass
+
+  @abstractmethod
+  def is_euicc(self) -> bool:
+    pass

openpilot/common/esim/esim.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import argparse
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.esim.base import LPABase, Profile
+
+
+def sorted_profiles(lpa: LPABase) -> list[Profile]:
+  return sorted(lpa.list_profiles(), key=lambda p: p.iccid)
+
+
+def resolve_iccid(lpa: LPABase, ref: str) -> str:
+  # ref is either a 1-based index into the sorted list, or a literal iccid
+  if ref.isdigit():
+    profiles = sorted_profiles(lpa)
+    idx = int(ref) - 1
+    if not 0 <= idx < len(profiles):
+      raise SystemExit(f'no profile at index {ref} (have {len(profiles)})')
+    return profiles[idx].iccid
+  return ref
+
+
+def print_profiles(lpa: LPABase) -> None:
+  profiles = sorted_profiles(lpa)
+  print(f'\n{len(profiles)} profile{"s" if len(profiles) != 1 else ""}:')
+  for i, p in enumerate(profiles, start=1):
+    print(f'{i}. {p.iccid} (nickname: {p.nickname or "<none provided>"}) (provider: {p.provider}) - {"enabled" if p.enabled else "disabled"}')
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(prog='esim.py', description='manage eSIM profiles on your comma device', epilog='comma.ai')
+  sub = parser.add_subparsers(dest='cmd')
+
+  sub.add_parser('list', help='list profiles')
+
+  p_switch = sub.add_parser('switch', help='switch to profile')
+  p_switch.add_argument('profile', help='iccid or 1-based index from `list`')
+
+  p_delete = sub.add_parser('delete', help='delete profile (warning: this cannot be undone)')
+  p_delete.add_argument('profile', help='iccid or 1-based index from `list`')
+
+  p_download = sub.add_parser('download', help='download a profile using QR code (format: LPA:1$rsp.truphone.com$QRF-SPEEDTEST)')
+  p_download.add_argument('qr')
+  p_download.add_argument('name')
+
+  p_nickname = sub.add_parser('nickname', help='update the nickname for a profile')
+  p_nickname.add_argument('profile', help='iccid or 1-based index from `list`')
+  p_nickname.add_argument('name')
+
+  args = parser.parse_args()
+
+  lpa = HARDWARE.get_sim_lpa()
+  if args.cmd == 'switch':
+    lpa.switch_profile(resolve_iccid(lpa, args.profile))
+  elif args.cmd == 'delete':
+    iccid = resolve_iccid(lpa, args.profile)
+    confirm = input(f'are you sure you want to delete profile {iccid}? (y/N) ')
+    if confirm == 'y':
+      lpa.delete_profile(iccid)
+    else:
+      print('cancelled')
+      exit(0)
+  elif args.cmd == 'download':
+    lpa.download_profile(args.qr, args.name)
+  elif args.cmd == 'nickname':
+    lpa.nickname_profile(resolve_iccid(lpa, args.profile), args.name)
+  else:
+    if args.cmd is None:
+      parser.print_help()
+    print_profiles(lpa)

openpilot/common/esim/gsma_ci_bundle.pem

@@ -0,0 +1,133 @@
+# GSMA Certificate Issuer (CI) bundle for eSIM RSP
+# Source: https://euicc-manual.osmocom.org/docs/pki/ci/bundle.pem
+
+issuer=
+    countryName = CH
+    organizationName = OISTE Foundation
+    commonName = OISTE GSMA CI G1
+notBefore=2024-01-16 23:17:39Z
+notAfter=2059-01-07 23:17:38Z
+-----BEGIN CERTIFICATE-----
+MIIB9zCCAZ2gAwIBAgIUSpBSCCDYPOEG/IFHUCKpZ2pIAQMwCgYIKoZIzj0EAwIw
+QzELMAkGA1UEBhMCQ0gxGTAXBgNVBAoMEE9JU1RFIEZvdW5kYXRpb24xGTAXBgNV
+BAMMEE9JU1RFIEdTTUEgQ0kgRzEwIBcNMjQwMTE2MjMxNzM5WhgPMjA1OTAxMDcy
+MzE3MzhaMEMxCzAJBgNVBAYTAkNIMRkwFwYDVQQKDBBPSVNURSBGb3VuZGF0aW9u
+MRkwFwYDVQQDDBBPSVNURSBHU01BIENJIEcxMFkwEwYHKoZIzj0CAQYIKoZIzj0D
+AQcDQgAEvZ3s3PFC4NgrCcCMmHJ6DJ66uzAHuLcvjJnOn+TtBNThS7YHLDyHCa2v
+7D+zTP+XTtgqgcLoB56Gha9EQQQ4xKNtMGswDwYDVR0TAQH/BAUwAwEB/zAQBgNV
+HREECTAHiAVghXQFDjAXBgNVHSABAf8EDTALMAkGB2eBEgECAQAwHQYDVR0OBBYE
+FEwnlnrSDBSzkelgHkHmBK1XwCIvMA4GA1UdDwEB/wQEAwIBBjAKBggqhkjOPQQD
+AgNIADBFAiBVcywTj017jKpAQ+gwy4MqK2hQvzve6lkvQkgSP6ykHwIhAI0KFwCD
+jnPbmcJsG41hUrWNlf+IcrMvFuYii0DasBNi
+-----END CERTIFICATE-----
+issuer=
+    organizationName = GSM Association
+    commonName = GSM Association - RSP2 Root CI1
+notBefore=2017-02-22 00:00:00Z
+notAfter=2052-02-21 23:59:59Z
+-----BEGIN CERTIFICATE-----
+MIICSTCCAe+gAwIBAgIQbmhWeneg7nyF7hg5Y9+qejAKBggqhkjOPQQDAjBEMRgw
+FgYDVQQKEw9HU00gQXNzb2NpYXRpb24xKDAmBgNVBAMTH0dTTSBBc3NvY2lhdGlv
+biAtIFJTUDIgUm9vdCBDSTEwIBcNMTcwMjIyMDAwMDAwWhgPMjA1MjAyMjEyMzU5
+NTlaMEQxGDAWBgNVBAoTD0dTTSBBc3NvY2lhdGlvbjEoMCYGA1UEAxMfR1NNIEFz
+c29jaWF0aW9uIC0gUlNQMiBSb290IENJMTBZMBMGByqGSM49AgEGCCqGSM49AwEH
+A0IABJ1qutL0HCMX52GJ6/jeibsAqZfULWj/X10p/Min6seZN+hf5llovbCNuB2n
+unLz+O8UD0SUCBUVo8e6n9X1TuajgcAwgb0wDgYDVR0PAQH/BAQDAgEGMA8GA1Ud
+EwEB/wQFMAMBAf8wEwYDVR0RBAwwCogIKwYBBAGC6WAwFwYDVR0gAQH/BA0wCzAJ
+BgdngRIBAgEAME0GA1UdHwRGMEQwQqBAoD6GPGh0dHA6Ly9nc21hLWNybC5zeW1h
+dXRoLmNvbS9vZmZsaW5lY2EvZ3NtYS1yc3AyLXJvb3QtY2kxLmNybDAdBgNVHQ4E
+FgQUgTcPUSXQsdQI1MOyMubSXnlb6/swCgYIKoZIzj0EAwIDSAAwRQIgIJdYsOMF
+WziPK7l8nh5mu0qiRiVf25oa9ullG/OIASwCIQDqCmDrYf+GziHXBOiwJwnBaeBO
+aFsiLzIEOaUuZwdNUw==
+-----END CERTIFICATE-----
+issuer=
+    countryName = US
+    organizationName = Entrust, Inc.
+    organizationalUnitName = See www.entrust.net/legal-terms
+    organizationalUnitName = (c) 2016 Entrust, Inc. - for authorized use only
+    commonName = Entrust eSIM Certification Authority
+notBefore=2016-11-16 16:04:02Z
+notAfter=2051-10-16 16:34:02Z
+-----BEGIN CERTIFICATE-----
+MIIC6DCCAo2gAwIBAgIRAIy4GT7M5nHsAAAAAFgsinowCgYIKoZIzj0EAwIwgbkx
+CzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9T
+ZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAx
+NiBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxLTArBgNV
+BAMTJEVudHJ1c3QgZVNJTSBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAgFw0xNjEx
+MTYxNjA0MDJaGA8yMDUxMTAxNjE2MzQwMlowgbkxCzAJBgNVBAYTAlVTMRYwFAYD
+VQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1c3QubmV0
+L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNiBFbnRydXN0LCBJbmMuIC0g
+Zm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxLTArBgNVBAMTJEVudHJ1c3QgZVNJTSBD
+ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA
+BAdzwGHeQ1Wb2f4DmHTByR5/IWL3JugQ1U3908a++bHdlt+TTA7K4c5cYZ+51Yz/
+hg/bacxguPDh9uQUK6Wg3a6jcjBwMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/
+BAQDAgEGMBcGA1UdIAEB/wQNMAswCQYHZ4ESAQIBADAVBgNVHREEDjAMiApghkgB
+hvpsFAoAMB0GA1UdDgQWBBQWcEt/NR42B/GMS3AAXDoAPf1BSjAKBggqhkjOPQQD
+AgNJADBGAiEAspjXMvaBZyAg86Z0AAtT0yBRAi1EyaAfNz9kDJeAE04CIQC3efj8
+ATL7/tDBOhANy3cK8PS/1NIlu9vqMLCZsZvJ0Q==
+-----END CERTIFICATE-----
+issuer=
+    countryName = FR
+    organizationName = OBERTHUR TECHNOLOGIES
+    organizationalUnitName = TELECOM
+    commonName = MC4 OT ROOT CI v1
+notBefore=2016-11-15 00:00:01Z
+notAfter=2046-11-08 23:59:59Z
+-----BEGIN CERTIFICATE-----
+MIICOjCCAeGgAwIBAgIBATAKBggqhkjOPQQDAjBbMQswCQYDVQQGEwJGUjEeMBwG
+A1UEChMVT0JFUlRIVVIgVEVDSE5PTE9HSUVTMRAwDgYDVQQLEwdURUxFQ09NMRow
+GAYDVQQDExFNQzQgT1QgUk9PVCBDSSB2MTAeFw0xNjExMTUwMDAwMDFaFw00NjEx
+MDgyMzU5NTlaMFsxCzAJBgNVBAYTAkZSMR4wHAYDVQQKExVPQkVSVEhVUiBURUNI
+Tk9MT0dJRVMxEDAOBgNVBAsTB1RFTEVDT00xGjAYBgNVBAMTEU1DNCBPVCBST09U
+IENJIHYxMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEHb/Gajt3OZxuaDSklBQE
+D4lOd6PGPLSvtfkM952ubdyy45tJwAeA0eEii0CLrFT6tcfXkW+H/5mQyMRXaAUk
+T6OBlTCBkjAfBgNVHSMEGDAWgBTNbmC3LXoGPLyEYluR6A/jBAbhPjAdBgNVHQ4E
+FgQUzW5gty16Bjy8hGJbkegP4wQG4T4wDgYDVR0PAQH/BAQDAgAGMBcGA1UdIAEB
+/wQNMAswCQYHZ4ESAQIBADAWBgNVHREEDzANiAsrBgEEAYHvb7OITTAPBgNVHRMB
+Af8EBTADAQH/MAoGCCqGSM49BAMCA0cAMEQCIEw4Nc7f2fDtoH+6ON/bknfDQxmT
+ikThXjhpLtSrSKN2AiAxHxgC87L0FDnH8dJNlkdGX9c0JIx6oLheIplfS6k+jg==
+-----END CERTIFICATE-----
+issuer=
+    commonName =  SubMan V4.2 CI Google Pixel
+    organizationName = Giesecke and Devrient GmbH
+    organizationalUnitName = Mobile Security
+    countryName = DE
+notBefore=2017-05-10 00:00:00Z
+notAfter=2027-05-10 00:00:00Z
+-----BEGIN CERTIFICATE-----
+MIICaTCCAg6gAwIBAgICASwwCgYIKoZIzj0EAwIwczElMCMGA1UEAxMcIFN1Yk1h
+biBWNC4yIENJIEdvb2dsZSBQaXhlbDEjMCEGA1UEChMaR2llc2Vja2UgYW5kIERl
+dnJpZW50IEdtYkgxGDAWBgNVBAsTD01vYmlsZSBTZWN1cml0eTELMAkGA1UEBhMC
+REUwHhcNMTcwNTEwMDAwMDAwWhcNMjcwNTEwMDAwMDAwWjBzMSUwIwYDVQQDExwg
+U3ViTWFuIFY0LjIgQ0kgR29vZ2xlIFBpeGVsMSMwIQYDVQQKExpHaWVzZWNrZSBh
+bmQgRGV2cmllbnQgR21iSDEYMBYGA1UECxMPTW9iaWxlIFNlY3VyaXR5MQswCQYD
+VQQGEwJERTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABHNorfaJsGzqWNawyAhl
+IAv9QL2/+b9RsUoso06t/dKX1MRr5CUJ51acvv5TAFhQKIml+dwLbFnV5aO+8W6Z
+wxajgZEwgY4wHwYDVR0jBBgwFoAUtg8LiX/WMLiM/tYWH46oCMU4KsMwHQYDVR0O
+BBYEFLYPC4l/1jC4jP7WFh+OqAjFOCrDMA4GA1UdDwEB/wQEAwIBBjAXBgNVHSAB
+Af8EDTALMAkGB2eBEgECAQAwDwYDVR0TAQH/BAUwAwEB/zASBgNVHREECzAJiAcr
+BgEEAdwPMAoGCCqGSM49BAMCA0kAMEYCIQDpoZcuAQrjATW8U+AWqMUJ0dY6nWW1
+R1QmFzVZ1yMXSwIhALCvRqkCtgiavdeFeSgsSNbY5Fhd+QoCltuSh1U4TE7A
+-----END CERTIFICATE-----
+issuer=
+    countryName = DE
+    commonName = SubMan V4.2 CI
+    organizationName = Giesecke and Devrient
+    organizationalUnitName = Mobile Security
+notBefore=2016-08-12 13:51:48Z
+notAfter=2026-08-12 13:51:48Z
+-----BEGIN CERTIFICATE-----
+MIICUjCCAfigAwIBAgIDQgAAMAoGCCqGSM49BAMCMGAxCzAJBgNVBAYTAkRFMRcw
+FQYDVQQDEw5TdWJNYW4gVjQuMiBDSTEeMBwGA1UEChMVR2llc2Vja2UgYW5kIERl
+dnJpZW50MRgwFgYDVQQLEw9Nb2JpbGUgU2VjdXJpdHkwHhcNMTYwODEyMTM1MTQ4
+WhcNMjYwODEyMTM1MTQ4WjBgMQswCQYDVQQGEwJERTEXMBUGA1UEAxMOU3ViTWFu
+IFY0LjIgQ0kxHjAcBgNVBAoTFUdpZXNlY2tlIGFuZCBEZXZyaWVudDEYMBYGA1UE
+CxMPTW9iaWxlIFNlY3VyaXR5MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYIgl
+VQr9wbXOlwPp8qMg5Df08Cli9Mc+lpr3Lwa9PlVA3QWlLeX4GfD4H3phLBqVIa17
+yHttmtheTxi0KoEqhKOBoDCBnTAdBgNVHQ4EFgQU6lOt7zMpuVCa/XVf1Ei4LcG8
+7P8wDgYDVR0PAQH/BAQDAgEGMBcGA1UdIAEB/wQNMAswCQYHZ4ESAQIBADAPBgNV
+HRMBAf8EBTADAQH/MBIGA1UdEQQLMAmIBysGAQQB3A8wLgYDVR0fBCcwJTAjoCGg
+H4YdaHR0cDovL2dpLWRlLmNvbS90ZXN0LmNybC5wZW0wCgYIKoZIzj0EAwIDSAAw
+RQIhAMMx2L/VHDiOW+Fl/OuFmhCdizYM17Yn9zAVieKO2T0iAiANWtCMmY+DzkqK
+yHxBFX0U2tBd682zP4DpgRt8j3Ylew==
+-----END CERTIFICATE-----

openpilot/common/esim/lpa.py

@@ -0,0 +1,791 @@
+# SGP.22 v2.3: https://www.gsma.com/solutions-and-impact/technologies/esim/wp-content/uploads/2021/07/SGP.22-v2.3.pdf
+
+import atexit
+import base64
+import fcntl
+import hashlib
+import os
+import requests
+import subprocess
+import sys
+import termios
+import time
+
+from collections.abc import Callable, Generator
+from contextlib import contextmanager
+from typing import Any
+
+from pathlib import Path
+
+from openpilot.common.time_helpers import system_time_valid
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+from openpilot.common.serial import Serial, SerialException
+
+GSMA_CI_BUNDLE = str(Path(__file__).parent / "gsma_ci_bundle.pem")
+
+DEFAULT_DEVICE = "/dev/modem_at0"
+DEFAULT_BAUD = 9600
+DEFAULT_TIMEOUT = 5.0
+# https://euicc-manual.osmocom.org/docs/lpa/applet-id/
+ISDR_AID = "A0000005591010FFFFFFFF8900000100"
+ES10X_MSS = 120
+HTTP_TIMEOUT = 30
+OPEN_ISDR_RETRIES = 10
+OPEN_ISDR_RETRY_DELAY_S = 0.25
+OPEN_ISDR_RESET_ATTEMPT = 5
+SEND_APDU_RETRIES = 3
+LOCK_FILE = '/dev/shm/modem.lock'
+DEBUG = os.environ.get("DEBUG") == "1"
+
+
+# TLV Tags
+TAG_ICCID = 0x5A
+TAG_STATUS = 0x80
+TAG_EUICC_INFO = 0xBF20
+TAG_PREPARE_DOWNLOAD = 0xBF21
+TAG_BPP_COMMAND = 0xBF23
+TAG_PROFILE_METADATA = 0xBF25
+TAG_INSTALL_RESULT_DATA = 0xBF27
+TAG_LIST_NOTIFICATION = 0xBF28
+TAG_SET_NICKNAME = 0xBF29
+TAG_RETRIEVE_NOTIFICATION = 0xBF2B
+TAG_PROFILE_INFO_LIST = 0xBF2D
+TAG_EUICC_CHALLENGE = 0xBF2E
+TAG_NOTIFICATION_METADATA = 0xBF2F
+TAG_NOTIFICATION_SENT = 0xBF30
+TAG_ENABLE_PROFILE = 0xBF31
+TAG_DELETE_PROFILE = 0xBF33
+TAG_BPP = 0xBF36
+TAG_PROFILE_INSTALL_RESULT = 0xBF37
+TAG_AUTH_SERVER = 0xBF38
+TAG_CANCEL_SESSION = 0xBF41
+TAG_OK = 0xA0
+
+PROFILE_OK = 0x00
+PROFILE_NOT_IN_DISABLED_STATE = 0x02
+PROFILE_CAT_BUSY = 0x05
+
+PROFILE_ERROR_CODES = {
+  0x01: "iccidOrAidNotFound", PROFILE_NOT_IN_DISABLED_STATE: "profileNotInDisabledState",
+  0x03: "disallowedByPolicy", 0x04: "wrongProfileReenabling",
+  PROFILE_CAT_BUSY: "catBusy", 0x06: "undefinedError",
+}
+AUTH_SERVER_ERROR_CODES = {
+  0x01: "eUICCVerificationFailed", 0x02: "eUICCCertificateExpired",
+  0x03: "eUICCCertificateRevoked", 0x05: "invalidServerSignature",
+  0x06: "euiccCiPKUnknown", 0x0A: "matchingIdRefused",
+  0x10: "insufficientMemory",
+}
+BPP_COMMAND_NAMES = {
+  0: "initialiseSecureChannel", 1: "configureISDP", 2: "storeMetadata",
+  3: "storeMetadata2", 4: "replaceSessionKeys", 5: "loadProfileElements",
+}
+BPP_ERROR_REASONS = {
+  1: "incorrectInputValues", 2: "invalidSignature", 3: "invalidTransactionId",
+  4: "unsupportedCrtValues", 5: "unsupportedRemoteOperationType",
+  6: "unsupportedProfileClass", 7: "scp03tStructureError", 8: "scp03tSecurityError",
+  9: "iccidAlreadyExistsOnEuicc", 10: "insufficientMemoryForProfile",
+  11: "installInterrupted", 12: "peProcessingError", 13: "dataMismatch",
+  14: "invalidNAA",
+}
+BPP_ERROR_MESSAGES = {
+  9: "This eSIM profile is already installed on this device.",
+  10: "Not enough memory on the eUICC to install this profile.",
+  12: "Profile installation failed. The QR code may have already been used.",
+}
+
+# SGP.22 §5.2.6 SM-DP+ reason/subject codes mapped to user-friendly messages
+ES9P_ERROR_MESSAGES: dict[tuple[str, str], str] = {
+  ('3.8', '8.2.6'): "This eSIM profile is already installed on another device. Please use a new QR code.",
+  ('3.8', '8.2.1'): "This eSIM profile has expired. Please request a new QR code.",
+  ('3.8', '8.1'): "The SM-DP+ server refused this request.",
+  ('3.1', '8.2.6'): "This eSIM profile has been revoked by the carrier.",
+  ('3.9', '8.2.6'): "This eSIM profile download has already been completed.",
+  ('2.1', '8.8'): "The device is not compatible with this eSIM profile.",
+  ('1.2', '8.1'): "The SM-DP+ server is temporarily unavailable. Try again later.",
+}
+
+NOTIFICATION_OPERATIONS = {0x80: "install", 0x40: "enable", 0x20: "disable", 0x10: "delete"}
+
+STATE_LABELS = {0: "disabled", 1: "enabled", 255: "unknown"}
+ICON_LABELS = {0: "jpeg", 1: "png", 255: "unknown"}
+CLASS_LABELS = {0: "test", 1: "provisioning", 2: "operational", 255: "unknown"}
+
+# TLV tag -> (field_name, decoder)
+FieldMap = dict[int, tuple[str, Callable[[bytes], Any]]]
+
+
+def b64e(data: bytes) -> str:
+  return base64.b64encode(data).decode("ascii")
+
+
+def base64_trim(s: str) -> str:
+  return "".join(c for c in s if c not in "\n\r \t")
+
+
+def b64d(s: str) -> bytes:
+  return base64.b64decode(base64_trim(s))
+
+
+class AtClient:
+  def __init__(self, device: str, baud: int, timeout: float) -> None:
+    self.channel: str | None = None
+    self._device = device
+    self._baud = baud
+    self._timeout = timeout
+    self._serial: Serial | None = None
+
+  def send_raw(self, data: bytes) -> None:
+    self._ensure_serial()
+    self._serial.reset_input_buffer()
+    self._serial.write(data)
+    self._serial.flush()
+
+  def close(self) -> None:
+    try:
+      if self.channel:
+        try:
+          self.query(f"AT+CCHC={self.channel}")
+        except (RuntimeError, TimeoutError):
+          pass
+        self.channel = None
+    finally:
+      if self._serial:
+        self._serial.close()
+
+  def _send(self, cmd: str) -> None:
+    if DEBUG:
+      print(f"SER >> {cmd}", file=sys.stderr)
+    self._serial.write((cmd + "\r").encode("ascii"))
+
+  def _expect(self) -> list[str]:
+    lines: list[str] = []
+    while True:
+      raw = self._serial.readline()
+      if not raw:
+        raise TimeoutError("AT command timed out")
+      line = raw.decode(errors="ignore").strip()
+      if not line:
+        continue
+      if DEBUG:
+        print(f"SER << {line}", file=sys.stderr)
+      if line == "OK":
+        return lines
+      if line == "ERROR" or line.startswith("+CME ERROR"):
+        raise RuntimeError(f"AT command failed: {line}")
+      lines.append(line)
+
+  def _ensure_serial(self, reconnect: bool = False) -> None:
+    if reconnect:
+      self.channel = None
+      try:
+        if self._serial:
+          self._serial.close()
+      except Exception:
+        pass
+      self._serial = None
+    if self._serial is None:
+      self._serial = Serial(self._device, baudrate=self._baud, timeout=self._timeout)
+
+  def query(self, cmd: str) -> list[str]:
+    self._ensure_serial()
+    try:
+      self._send(cmd)
+      return self._expect()
+    except SerialException:
+      self._ensure_serial(reconnect=True)
+      self._send(cmd)
+      return self._expect()
+
+  def _open_isdr_once(self) -> None:
+    if self.channel:
+      try:
+        self.query(f"AT+CCHC={self.channel}")
+      except RuntimeError:
+        pass
+      self.channel = None
+    # drain any unsolicited responses before opening
+    if self._serial:
+      try:
+        self._serial.reset_input_buffer()
+      except (OSError, SerialException, termios.error):
+        self._ensure_serial(reconnect=True)
+    for line in self.query(f'AT+CCHO="{ISDR_AID}"'):
+      if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
+        self.channel = ch
+        return
+    raise RuntimeError("Failed to open ISD-R application")
+
+  def _reset_modem(self) -> None:
+    if self._serial:
+      try:
+        self._serial.close()
+      except Exception:
+        pass
+      self._serial = None
+    subprocess.run(['/usr/comma/lte/lte.sh', 'start'], capture_output=True)
+
+  def open_isdr(self) -> None:
+    for attempt in range(OPEN_ISDR_RETRIES):
+      try:
+        self._open_isdr_once()
+        return
+      except (RuntimeError, TimeoutError, termios.error, SerialException):
+        time.sleep(OPEN_ISDR_RETRY_DELAY_S)
+        if attempt == OPEN_ISDR_RESET_ATTEMPT:
+          self._reset_modem()
+    raise RuntimeError("Failed to open ISD-R after retries")
+
+  def send_apdu(self, apdu: bytes) -> tuple[bytes, int, int]:
+    for attempt in range(SEND_APDU_RETRIES):
+      try:
+        if not self.channel:
+          self.open_isdr()
+        hex_payload = apdu.hex().upper()
+        for line in self.query(f'AT+CGLA={self.channel},{len(hex_payload)},"{hex_payload}"'):
+          if line.startswith("+CGLA:"):
+            parts = line.split(":", 1)[1].split(",", 1)
+            if len(parts) == 2:
+              data = bytes.fromhex(parts[1].strip().strip('"'))
+              if len(data) >= 2:
+                return data[:-2], data[-2], data[-1]
+        raise RuntimeError("Missing +CGLA response")
+      except (RuntimeError, ValueError):
+        self.channel = None
+        if attempt == SEND_APDU_RETRIES - 1:
+          raise
+    raise RuntimeError("send_apdu failed")
+
+
+# --- TLV utilities ---
+
+def iter_tlv(data: bytes, with_positions: bool = False) -> Generator:
+  idx, length = 0, len(data)
+  while idx < length:
+    start_pos = idx
+    tag = data[idx]
+    idx += 1
+    if tag & 0x1F == 0x1F:  # Multi-byte tag
+      tag_value = tag
+      while idx < length:
+        next_byte = data[idx]
+        idx += 1
+        tag_value = (tag_value << 8) | next_byte
+        if not (next_byte & 0x80):
+          break
+    else:
+      tag_value = tag
+    if idx >= length:
+      break
+    size = data[idx]
+    idx += 1
+    if size & 0x80:  # Multi-byte length
+      num_bytes = size & 0x7F
+      if idx + num_bytes > length:
+        break
+      size = int.from_bytes(data[idx : idx + num_bytes], "big")
+      idx += num_bytes
+    if idx + size > length:
+      break
+    value = data[idx : idx + size]
+    idx += size
+    yield (tag_value, value, start_pos, idx) if with_positions else (tag_value, value)
+
+
+def find_tag(data: bytes, target: int) -> bytes | None:
+  return next((v for t, v in iter_tlv(data) if t == target), None)
+
+
+def require_tag(data: bytes, target: int, label: str = "") -> bytes:
+  v = find_tag(data, target)
+  if v is None:
+    raise RuntimeError(f"Missing {label or f'tag 0x{target:X}'}")
+  return v
+
+
+def tbcd_to_string(raw: bytes) -> str:
+  return "".join(str(n) for b in raw for n in (b & 0x0F, b >> 4) if n <= 9)
+
+
+def string_to_tbcd(s: str) -> bytes:
+  digits = [int(c) for c in s if c.isdigit()]
+  return bytes(digits[i] | ((digits[i + 1] if i + 1 < len(digits) else 0xF) << 4) for i in range(0, len(digits), 2))
+
+
+def encode_tlv(tag: int, value: bytes) -> bytes:
+  tag_bytes = bytes([(tag >> 8) & 0xFF, tag & 0xFF]) if tag > 255 else bytes([tag])
+  vlen = len(value)
+  if vlen <= 127:
+    return tag_bytes + bytes([vlen]) + value
+  length_bytes = vlen.to_bytes((vlen.bit_length() + 7) // 8, "big")
+  return tag_bytes + bytes([0x80 | len(length_bytes)]) + length_bytes + value
+
+
+def int_bytes(n: int) -> bytes:
+  """Encode a positive integer as minimal big-endian bytes (at least 1 byte)."""
+  return n.to_bytes((n.bit_length() + 7) // 8 or 1, "big")
+
+
+PROFILE: FieldMap = {
+  TAG_ICCID: ("iccid", tbcd_to_string),
+  0x4F: ("isdpAid", lambda v: v.hex().upper()),
+  0x9F70: ("profileState", lambda v: STATE_LABELS.get(v[0], "unknown")),
+  0x90: ("profileNickname", lambda v: v.decode("utf-8", errors="ignore") or None),
+  0x91: ("serviceProviderName", lambda v: v.decode("utf-8", errors="ignore") or None),
+  0x92: ("profileName", lambda v: v.decode("utf-8", errors="ignore") or None),
+  0x93: ("iconType", lambda v: ICON_LABELS.get(v[0], "unknown")),
+  0x94: ("icon", b64e),
+  0x95: ("profileClass", lambda v: CLASS_LABELS.get(v[0], "unknown")),
+}
+
+
+def decode_struct(data: bytes, field_map: FieldMap) -> dict[str, Any]:
+  """Parse TLV data using a {tag: (field_name, decoder)} map into a dict."""
+  result: dict[str, Any] = {name: None for name, _ in field_map.values()}
+  for tag, value in iter_tlv(data):
+    if (field := field_map.get(tag)):
+      result[field[0]] = field[1](value)
+  return result
+
+
+# --- ES10x command transport ---
+
+def es10x_command(client: AtClient, data: bytes) -> bytes:
+  response = bytearray()
+  sequence = 0
+  offset = 0
+  while offset < len(data):
+    chunk = data[offset : offset + ES10X_MSS]
+    offset += len(chunk)
+    is_last = offset == len(data)
+    apdu = bytes([0x80, 0xE2, 0x91 if is_last else 0x11, sequence & 0xFF, len(chunk)]) + chunk
+    segment, sw1, sw2 = client.send_apdu(apdu)
+    response.extend(segment)
+    while True:
+      if sw1 == 0x61:  # More data available
+        segment, sw1, sw2 = client.send_apdu(bytes([0x80, 0xC0, 0x00, 0x00, sw2 or 0]))
+        response.extend(segment)
+        continue
+      if (sw1 & 0xF0) == 0x90:
+        break
+      raise RuntimeError(f"APDU failed with SW={sw1:02X}{sw2:02X}")
+    sequence += 1
+  return bytes(response)
+
+
+# --- Profile operations ---
+
+NOTIFICATION: FieldMap = {
+  TAG_STATUS: ("seqNumber", lambda v: int.from_bytes(v, "big")),
+  0x81: ("profileManagementOperation",
+         lambda v: NOTIFICATION_OPERATIONS.get(next((m for m in NOTIFICATION_OPERATIONS if len(v) >= 2 and v[1] & m), 0), "unknown")),
+  0x0C: ("notificationAddress", lambda v: v.decode("utf-8", errors="ignore")),
+  TAG_ICCID: ("iccid", tbcd_to_string),
+}
+
+
+def decode_profiles(blob: bytes) -> list[dict]:
+  root = require_tag(blob, TAG_PROFILE_INFO_LIST, "ProfileInfoList")
+  list_ok = find_tag(root, TAG_OK)
+  if list_ok is None:
+    return []
+  return [decode_struct(value, PROFILE) for tag, value in iter_tlv(list_ok) if tag == 0xE3]
+
+
+def list_profiles(client: AtClient) -> list[dict]:
+  return decode_profiles(es10x_command(client, TAG_PROFILE_INFO_LIST.to_bytes(2, "big") + b"\x00"))
+
+
+def set_profile_nickname(client: AtClient, iccid: str, nickname: str) -> None:
+  nickname_bytes = nickname.encode("utf-8")
+  if len(nickname_bytes) > 64:
+    raise ValueError("Profile nickname must be 64 bytes or less")
+  content = encode_tlv(TAG_ICCID, string_to_tbcd(iccid)) + encode_tlv(0x90, nickname_bytes)
+  response = es10x_command(client, encode_tlv(TAG_SET_NICKNAME, content))
+  code = require_tag(require_tag(response, TAG_SET_NICKNAME, "SetNicknameResponse"), TAG_STATUS, "SetNickname status")[0]
+  if code == 0x01:
+    raise LPAError(f"profile {iccid} not found")
+  if code != 0x00:
+    raise RuntimeError(f"SetNickname failed with status 0x{code:02X}")
+
+
+# --- ES9P HTTP ---
+
+def es9p_request(smdp_address: str, endpoint: str, payload: dict, error_prefix: str = "Request", session: requests.Session | None = None) -> dict:
+  url = f"https://{smdp_address}/gsma/rsp2/es9plus/{endpoint}"
+  headers = {"User-Agent": "gsma-rsp-lpad", "X-Admin-Protocol": "gsma/rsp/v2.3.0", "Content-Type": "application/json"}
+  http = session or requests
+  resp = http.post(url, json=payload, headers=headers, timeout=HTTP_TIMEOUT, verify=GSMA_CI_BUNDLE)
+  resp.raise_for_status()
+  if not resp.content:
+    return {}
+  data = resp.json()
+  if "header" in data and "functionExecutionStatus" in data["header"]:
+    status = data["header"]["functionExecutionStatus"]
+    if status.get("status") == "Failed":
+      sd = status.get("statusCodeData", {})
+      reason = sd.get("reasonCode", "unknown")
+      subject = sd.get("subjectCode", "unknown")
+      msg = ES9P_ERROR_MESSAGES.get((reason, subject),
+            f"{error_prefix} failed: {reason}/{subject} - {sd.get('message', 'unknown')}")
+      raise RuntimeError(msg)
+  return data
+
+
+# --- Notifications ---
+
+def list_notifications(client: AtClient) -> list[dict]:
+  response = es10x_command(client, encode_tlv(TAG_LIST_NOTIFICATION, b""))
+  root = require_tag(response, TAG_LIST_NOTIFICATION, "ListNotificationResponse")
+  metadata_list = find_tag(root, TAG_OK)
+  if metadata_list is None:
+    return []
+  return [decode_struct(value, NOTIFICATION) for tag, value in iter_tlv(metadata_list) if tag == TAG_NOTIFICATION_METADATA]
+
+
+def process_notifications(client: AtClient) -> None:
+  for notification in list_notifications(client):
+    seq_number, smdp_address = notification["seqNumber"], notification["notificationAddress"]
+    try:
+      request = encode_tlv(TAG_RETRIEVE_NOTIFICATION, encode_tlv(TAG_OK, encode_tlv(TAG_STATUS, int_bytes(seq_number))))
+      response = es10x_command(client, request)
+      content = require_tag(require_tag(response, TAG_RETRIEVE_NOTIFICATION, "RetrieveNotificationsListResponse"),
+                            TAG_OK, "RetrieveNotificationsListResponse")
+      pending_notif = next((v for t, v in iter_tlv(content) if t in (TAG_PROFILE_INSTALL_RESULT, 0x30)), None)
+      if pending_notif is None:
+        raise RuntimeError("Missing PendingNotification")
+
+      es9p_request(smdp_address, "handleNotification", {"pendingNotification": b64e(pending_notif)}, "HandleNotification")
+
+      response = es10x_command(client, encode_tlv(TAG_NOTIFICATION_SENT, encode_tlv(TAG_STATUS, int_bytes(seq_number))))
+      root = require_tag(response, TAG_NOTIFICATION_SENT, "NotificationSentResponse")
+      if int.from_bytes(require_tag(root, TAG_STATUS, "RemoveNotificationFromList status"), "big") != 0:
+        raise RuntimeError("RemoveNotificationFromList failed")
+    except Exception as e:
+      print(f"notification {seq_number} failed: {e}", file=sys.stderr)
+
+
+# --- Authentication & Download ---
+
+def get_challenge_and_info(client: AtClient) -> tuple[bytes, bytes]:
+  challenge_resp = es10x_command(client, encode_tlv(TAG_EUICC_CHALLENGE, b""))
+  challenge = require_tag(require_tag(challenge_resp, TAG_EUICC_CHALLENGE, "GetEuiccDataResponse"),
+                          TAG_STATUS, "challenge in response")
+  info_resp = es10x_command(client, encode_tlv(TAG_EUICC_INFO, b""))
+  require_tag(info_resp, TAG_EUICC_INFO, "GetEuiccInfo1Response")
+  return challenge, info_resp
+
+
+def authenticate_server(client: AtClient, b64_signed1: str, b64_sig1: str, b64_pk_id: str, b64_cert: str, matching_id: str) -> str:
+  tac = bytes([0x35, 0x29, 0x06, 0x11])
+  device_info = encode_tlv(TAG_STATUS, tac) + encode_tlv(0xA1, b"")
+  ctx_inner = encode_tlv(TAG_STATUS, matching_id.encode("utf-8")) + encode_tlv(0xA1, device_info)
+  content = b64d(b64_signed1) + b64d(b64_sig1) + b64d(b64_pk_id) + b64d(b64_cert) + encode_tlv(0xA0, ctx_inner)
+  response = es10x_command(client, encode_tlv(TAG_AUTH_SERVER, content))
+  root = require_tag(response, TAG_AUTH_SERVER, "AuthenticateServerResponse")
+  error_tag = find_tag(root, 0xA1)
+  if error_tag is not None:
+    code = int.from_bytes(error_tag, "big") if error_tag else 0
+    raise RuntimeError(f"AuthenticateServer rejected by eUICC: {AUTH_SERVER_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
+  return b64e(response)
+
+
+def prepare_download(client: AtClient, b64_signed2: str, b64_sig2: str, b64_cert: str, cc: str | None = None) -> str:
+  smdp_signed2 = b64d(b64_signed2)
+  smdp_signature2 = b64d(b64_sig2)
+  smdp_certificate = b64d(b64_cert)
+  smdp_signed2_root = find_tag(smdp_signed2, 0x30)
+  if smdp_signed2_root is None:
+    raise RuntimeError("Invalid smdpSigned2")
+  transaction_id = find_tag(smdp_signed2_root, TAG_STATUS)
+  cc_required_flag = find_tag(smdp_signed2_root, 0x01)
+  if transaction_id is None or cc_required_flag is None:
+    raise RuntimeError("Invalid smdpSigned2")
+  content = smdp_signed2 + smdp_signature2
+  if int.from_bytes(cc_required_flag, "big") != 0:
+    if not cc:
+      raise RuntimeError("Confirmation code required but not provided")
+    content += encode_tlv(0x04, hashlib.sha256(hashlib.sha256(cc.encode("utf-8")).digest() + transaction_id).digest())
+  content += smdp_certificate
+  response = es10x_command(client, encode_tlv(TAG_PREPARE_DOWNLOAD, content))
+  require_tag(response, TAG_PREPARE_DOWNLOAD, "PrepareDownloadResponse")
+  return b64e(response)
+
+
+def _parse_tlv_header_len(data: bytes) -> int:
+  tag_len = 2 if data[0] & 0x1F == 0x1F else 1
+  length_byte = data[tag_len]
+  return tag_len + (1 + (length_byte & 0x7F) if length_byte & 0x80 else 1)
+
+
+def _split_bpp(bpp: bytes) -> list[bytes]:
+  """Split a BoundProfilePackage into APDU chunks per SGP.22 §5.7.6."""
+  root_value = None
+  for tag, value, start, end in iter_tlv(bpp, with_positions=True):
+    if tag == TAG_BPP:
+      root_value = value
+      val_start = start + _parse_tlv_header_len(bpp[start:end])
+      break
+  if root_value is None:
+    raise RuntimeError("Invalid BoundProfilePackage")
+
+  chunks: list[bytes] = []
+  for tag, value, start, end in iter_tlv(root_value, with_positions=True):
+    if tag == TAG_BPP_COMMAND:
+      chunks.append(bpp[0 : val_start + end])
+    elif tag in (0xA0, 0xA2):
+      chunks.append(bpp[val_start + start : val_start + end])
+    elif tag in (0xA1, 0xA3):
+      hdr_len = _parse_tlv_header_len(root_value[start:end])
+      chunks.append(bpp[val_start + start : val_start + start + hdr_len])
+      for _, _, cs, ce in iter_tlv(value, with_positions=True):
+        chunks.append(value[cs:ce])
+  return chunks
+
+
+def _parse_install_result(response: bytes) -> dict[str, Any] | None:
+  """Parse a ProfileInstallResult from an APDU response, or None if not present."""
+  root = find_tag(response, TAG_PROFILE_INSTALL_RESULT)
+  if not root:
+    return None
+  result_data = find_tag(root, TAG_INSTALL_RESULT_DATA)
+  if not result_data:
+    return None
+  result: dict[str, Any] = {"seqNumber": 0, "success": False, "bppCommandId": None, "errorReason": None}
+  notif_meta = find_tag(result_data, TAG_NOTIFICATION_METADATA)
+  if notif_meta:
+    seq_num = find_tag(notif_meta, TAG_STATUS)
+    if seq_num:
+      result["seqNumber"] = int.from_bytes(seq_num, "big")
+  final_result = find_tag(result_data, 0xA2)
+  if final_result:
+    for tag, value in iter_tlv(final_result):
+      if tag == 0xA0:
+        result["success"] = True
+      elif tag == 0xA1:
+        bpp_cmd = find_tag(value, TAG_STATUS)
+        if bpp_cmd:
+          result["bppCommandId"] = int.from_bytes(bpp_cmd, "big")
+        err = find_tag(value, 0x81)
+        if err:
+          result["errorReason"] = int.from_bytes(err, "big")
+  return result
+
+
+def load_bpp(client: AtClient, b64_bpp: str) -> dict:
+  bpp = b64d(b64_bpp)
+  result = None
+  for chunk in _split_bpp(bpp):
+    response = es10x_command(client, chunk)
+    if response and (parsed := _parse_install_result(response)):
+      result = parsed
+      break
+
+  if result is None:
+    raise RuntimeError("Profile installation failed: no result from eUICC")
+  if not result["success"] and result["errorReason"] is not None:
+    msg = BPP_ERROR_MESSAGES.get(result["errorReason"])
+    if not msg:
+      cmd_name = BPP_COMMAND_NAMES.get(result["bppCommandId"], f"unknown({result['bppCommandId']})")
+      err_name = BPP_ERROR_REASONS.get(result["errorReason"], f"unknown({result['errorReason']})")
+      msg = f"Profile installation failed at {cmd_name}: {err_name}"
+    raise RuntimeError(msg)
+  if not result["success"]:
+    raise RuntimeError("Profile installation failed: no result from eUICC")
+  return result
+
+
+def parse_metadata(b64_metadata: str) -> dict:
+  root = find_tag(b64d(b64_metadata), TAG_PROFILE_METADATA)
+  if root is None:
+    raise RuntimeError("Invalid profileMetadata")
+  return decode_struct(root, PROFILE)
+
+
+def cancel_session(client: AtClient, transaction_id: bytes, reason: int = 127) -> str:
+  content = encode_tlv(0x80, transaction_id) + encode_tlv(0x81, bytes([reason]))
+  response = es10x_command(client, encode_tlv(TAG_CANCEL_SESSION, content))
+  return b64e(response)
+
+
+def parse_lpa_activation_code(activation_code: str) -> tuple[str, str]:
+  """Parse 'LPA:1$smdp.example.com$MATCHING-ID' into (smdp_address, matching_id)."""
+  if not activation_code.startswith("LPA:"):
+    raise ValueError("Invalid activation code format")
+  parts = activation_code[4:].split("$")
+  if len(parts) != 3:
+    raise ValueError("Invalid activation code format")
+  return parts[1], parts[2]
+
+
+def _b64_field(data: dict, key: str) -> str:
+  return base64_trim(data[key])
+
+
+def _cancel_session_safe(client: AtClient, smdp: str, tx_id: str, session: requests.Session) -> None:
+  b64_cancel = ""
+  try:
+    b64_cancel = cancel_session(client, b64d(tx_id))
+  except Exception:
+    pass
+  try:
+    es9p_request(smdp, "cancelSession", {"transactionId": tx_id, "cancelSessionResponse": b64_cancel}, "CancelSession", session=session)
+  except Exception:
+    pass
+
+
+def download_profile(client: AtClient, activation_code: str) -> str:
+  """Download and install an eSIM profile. Returns the ICCID of the installed profile."""
+  if not system_time_valid():
+    raise RuntimeError("System time is not set; TLS certificate validation requires a valid clock")
+  smdp, matching_id = parse_lpa_activation_code(activation_code)
+  challenge, euicc_info = get_challenge_and_info(client)
+  session = requests.Session()
+  tx_id = None
+
+  try:
+    # step 1: initiate authentication
+    auth = es9p_request(smdp, "initiateAuthentication", {
+      "smdpAddress": smdp, "euiccChallenge": b64e(challenge),
+      "euiccInfo1": b64e(euicc_info), "matchingId": matching_id,
+    }, "Authentication", session=session)
+    tx_id = _b64_field(auth, "transactionId")
+
+    # step 2: authenticate server
+    b64_auth = authenticate_server(client,
+      _b64_field(auth, "serverSigned1"), _b64_field(auth, "serverSignature1"),
+      _b64_field(auth, "euiccCiPKIdToBeUsed"), _b64_field(auth, "serverCertificate"),
+      matching_id)
+
+    # step 3: authenticate client + get metadata
+    cli = es9p_request(smdp, "authenticateClient", {
+      "transactionId": tx_id, "authenticateServerResponse": b64_auth,
+    }, "Authentication", session=session)
+    iccid = parse_metadata(_b64_field(cli, "profileMetadata"))["iccid"]
+
+    # step 4: prepare download
+    b64_prep = prepare_download(client,
+      _b64_field(cli, "smdpSigned2"), _b64_field(cli, "smdpSignature2"),
+      _b64_field(cli, "smdpCertificate"))
+
+    # step 5: get and install bound profile package
+    bpp = es9p_request(smdp, "getBoundProfilePackage", {
+      "transactionId": tx_id, "prepareDownloadResponse": b64_prep,
+    }, "GetBoundProfilePackage", session=session)
+    load_bpp(client, _b64_field(bpp, "boundProfilePackage"))
+    return iccid
+  except Exception:
+    if tx_id:
+      _cancel_session_safe(client, smdp, tx_id, session)
+    raise
+  finally:
+    session.close()
+
+
+class TiciLPA(LPABase):
+  def __init__(self):
+    if hasattr(self, '_client'):
+      return
+    self._client = AtClient(DEFAULT_DEVICE, DEFAULT_BAUD, DEFAULT_TIMEOUT)
+    atexit.register(self._client.close)
+
+  @contextmanager
+  def _acquire_lock(self):
+    fd = os.open(LOCK_FILE, os.O_CREAT | os.O_RDWR)
+    try:
+      fcntl.flock(fd, fcntl.LOCK_EX)
+      yield
+    finally:
+      fcntl.flock(fd, fcntl.LOCK_UN)
+      os.close(fd)
+
+  @contextmanager
+  def _acquire_channel(self):
+    with self._acquire_lock():
+      try:
+        self._client.open_isdr()
+        yield
+      finally:
+        if self._client.channel:
+          try:
+            self._client.query(f"AT+CCHC={self._client.channel}")
+          except (RuntimeError, TimeoutError):
+            pass
+          self._client.channel = None
+
+  def list_profiles(self) -> list[Profile]:
+    with self._acquire_channel():
+      return [
+        Profile(
+          iccid=p.get("iccid", ""),
+          nickname=p.get("profileNickname") or "",
+          enabled=p.get("profileState") == "enabled",
+          provider=p.get("serviceProviderName") or "",
+        )
+        for p in list_profiles(self._client)
+      ]
+
+  def get_active_profile(self) -> Profile | None:
+    return None
+
+  def process_notifications(self) -> None:
+    if not system_time_valid():
+      raise RuntimeError("System time is not set; TLS certificate validation requires a valid clock")
+    with self._acquire_channel():
+      process_notifications(self._client)
+
+  def delete_profile(self, iccid: str) -> None:
+    profile = next((p for p in self.list_profiles() if p.iccid == iccid), None)
+    if profile is None:
+      raise LPAProfileNotFoundError(f"profile not found: {iccid}")
+    if profile.is_comma:
+      raise LPAError("refusing to delete a comma profile")
+    with self._acquire_channel():
+      request = encode_tlv(TAG_DELETE_PROFILE, encode_tlv(TAG_ICCID, string_to_tbcd(iccid)))
+      response = es10x_command(self._client, request)
+      code = require_tag(require_tag(response, TAG_DELETE_PROFILE, "DeleteProfileResponse"), TAG_STATUS, "DeleteProfile status")[0]
+    if code != PROFILE_OK:
+      raise LPAError(f"DeleteProfile failed: {PROFILE_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
+
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    with self._acquire_channel():
+      iccid = download_profile(self._client, qr)
+      if nickname and iccid:
+        set_profile_nickname(self._client, iccid, nickname)
+
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    with self._acquire_channel():
+      set_profile_nickname(self._client, iccid, nickname)
+
+  def _enable_profile(self, iccid: str) -> int:
+    inner = encode_tlv(TAG_OK, encode_tlv(TAG_ICCID, string_to_tbcd(iccid)))
+    inner += b'\x01\x01\x01'  # refreshFlag=1
+    response = es10x_command(self._client, encode_tlv(TAG_ENABLE_PROFILE, inner))
+    return require_tag(require_tag(response, TAG_ENABLE_PROFILE, "EnableProfileResponse"), TAG_STATUS, "EnableProfile status")[0]
+
+  def switch_profile(self, iccid: str) -> None:
+    with self._acquire_channel():
+      code = self._enable_profile(iccid)
+      if code == PROFILE_CAT_BUSY:  # stale eUICC transaction, reset and retry
+        self._client._reset_modem()
+        self._client.open_isdr()
+        code = self._enable_profile(iccid)
+      if code not in (PROFILE_OK, PROFILE_NOT_IN_DISABLED_STATE):
+        raise LPAError(f"EnableProfile failed: {PROFILE_ERROR_CODES.get(code, 'unknown')} (0x{code:02X})")
+
+  def is_euicc(self) -> bool:
+    # +CCHO:<n> -> ISD-R applet present, eUICC. Any error -> non-eUICC.
+    with self._acquire_lock():
+      try:
+        lines = self._client.query(f'AT+CCHO="{ISDR_AID}"')
+      except RuntimeError:
+        return False
+      for line in lines:
+        if line.startswith("+CCHO:") and (ch := line.split(":", 1)[1].strip()):
+          try:
+            self._client.query(f"AT+CCHC={ch}")
+          except (RuntimeError, TimeoutError):
+            pass
+          self._client.channel = None
+          return True
+      return False

openpilot/common/file_chunker.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+import io
+import sys
+import math
+import os
+from pathlib import Path
+
+CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
+
+def get_chunk_name(name, idx, num_chunks):
+  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
+
+def get_manifest_path(name):
+  return f"{name}.chunkmanifest"
+
+def _chunk_paths(path, num_chunks):
+  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+
+def get_chunk_targets(path, file_size):
+  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+  return _chunk_paths(path, num_chunks)
+
+def chunk_file(path, targets):
+  manifest_path, *chunk_paths = targets
+  actual_num_chunks = max(1, math.ceil(os.path.getsize(path) / CHUNK_SIZE))
+  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
+  with open(path, 'rb') as f:
+    for chunk_path in chunk_paths:
+      with open(chunk_path, 'wb') as out:
+        out.write(f.read(CHUNK_SIZE))
+  Path(manifest_path).write_text(str(len(chunk_paths)))
+  os.remove(path)
+
+def get_existing_chunks(path):
+  if os.path.isfile(path):
+    return [path]
+  if os.path.isfile(manifest := get_manifest_path(path)):
+    num_chunks = int(Path(manifest).read_text().strip())
+    return _chunk_paths(path, num_chunks)
+  raise FileNotFoundError(path)
+
+class ChunkStream(io.RawIOBase):
+  def __init__(self, paths):
+    self._paths = iter(paths)
+    self._buf = memoryview(b'')
+
+  def readable(self):
+    return True
+
+  def readinto(self, b):
+    n = 0
+    while n < len(b):
+      if not self._buf:
+        p = next(self._paths, None)
+        if p is None:
+          break
+        with open(p, 'rb') as f:
+          self._buf = memoryview(f.read())
+        continue
+      take = min(len(b) - n, len(self._buf))
+      b[n:n + take] = self._buf[:take]
+      self._buf = self._buf[take:]
+      n += take
+    return n
+
+def open_file_chunked(path):
+  manifest_path = get_manifest_path(path)
+  if os.path.isfile(manifest_path):
+    num_chunks = int(Path(manifest_path).read_text().strip())
+    paths = [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+  elif os.path.isfile(path):
+    paths = [path]
+  else:
+    raise FileNotFoundError(path)
+  return io.BufferedReader(ChunkStream(paths))
+
+
+if __name__ == "__main__":
+  path = sys.argv[1]
+  chunk_paths = get_chunk_targets(path, os.path.getsize(path))
+  chunk_file(path, chunk_paths)

openpilot/common/filter_simple.py

@@ -0,0 +1,34 @@
+class FirstOrderFilter:
+  def __init__(self, x0, rc, dt, initialized=True):
+    self.x = x0
+    self.dt = dt
+    self.update_alpha(rc)
+    self.initialized = initialized
+
+  def update_alpha(self, rc):
+    self.alpha = self.dt / (rc + self.dt)
+
+  def update(self, x):
+    if self.initialized:
+      self.x = (1. - self.alpha) * self.x + self.alpha * x
+    else:
+      self.initialized = True
+      self.x = x
+    return self.x
+
+
+class BounceFilter(FirstOrderFilter):
+  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
+    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
+    self.bounce = bounce
+    super().__init__(x0, rc, dt, initialized)
+
+  def update(self, x):
+    super().update(x)
+    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
+    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
+    self.velocity.update(0.0)
+    if abs(self.velocity.x) < 1e-3:
+      self.velocity.x = 0.0
+    self.x += self.velocity.x
+    return self.x

openpilot/common/git.py

@@ -0,0 +1,42 @@
+from functools import cache
+import subprocess
+from openpilot.common.utils import run_cmd, run_cmd_default
+
+
+@cache
+def get_commit(cwd: str | None = None, branch: str = "HEAD") -> str:
+  return run_cmd_default(["git", "rev-parse", branch], cwd=cwd)
+
+
+@cache
+def get_commit_date(cwd: str | None = None, commit: str = "HEAD") -> str:
+  return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd)
+
+
+@cache
+def get_short_branch(cwd: str | None = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd)
+
+
+@cache
+def get_branch(cwd: str | None = None) -> str:
+  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd)
+
+
+@cache
+def get_origin(cwd: str | None = None) -> str:
+  try:
+    local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd)
+    tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd)
+    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
+  except subprocess.CalledProcessError:  # Not on a branch, fallback
+    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
+
+
+@cache
+def get_normalized_origin(cwd: str | None = None) -> str:
+  return get_origin(cwd) \
+    .replace("git@", "", 1) \
+    .replace(".git", "", 1) \
+    .replace("https://", "", 1) \
+    .replace(":", "/", 1)

openpilot/common/gpio.py

@@ -0,0 +1,89 @@
+import os
+import fcntl
+import ctypes
+from functools import cache
+
+def gpio_init(pin: int, output: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
+      f.write(b"out" if output else b"in")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} direction: {e}")
+
+def gpio_set(pin: int, high: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
+      f.write(b"1" if high else b"0")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+def gpio_read(pin: int) -> bool | None:
+  val = None
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
+      val = bool(int(f.read().strip()))
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+  return val
+
+def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
+  try:
+    with open("/sys/class/gpio/export", 'w') as f:
+      f.write(str(pin))
+  except Exception:
+    print(f"Failed to export gpio {pin}")
+
+@cache
+def get_irq_action(irq: int) -> list[str]:
+  try:
+    with open(f"/sys/kernel/irq/{irq}/actions") as f:
+      actions = f.read().strip().split(',')
+      return actions
+  except FileNotFoundError:
+    return []
+
+def get_irqs_for_action(action: str) -> list[str]:
+  ret = []
+  with open("/proc/interrupts") as f:
+    for l in f.readlines():
+      irq = l.split(':')[0].strip()
+      if irq.isdigit() and action in get_irq_action(irq):
+        ret.append(irq)
+  return ret
+
+# *** gpiochip ***
+
+class gpioevent_data(ctypes.Structure):
+  _fields_ = [
+    ("timestamp", ctypes.c_uint64),
+    ("id", ctypes.c_uint32),
+  ]
+
+class gpioevent_request(ctypes.Structure):
+  _fields_ = [
+    ("lineoffset", ctypes.c_uint32),
+    ("handleflags", ctypes.c_uint32),
+    ("eventflags", ctypes.c_uint32),
+    ("label", ctypes.c_char * 32),
+    ("fd", ctypes.c_int)
+  ]
+
+def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
+  GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
+  GPIOHANDLE_REQUEST_INPUT = 0x1
+  GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
+
+  rq = gpioevent_request()
+  rq.lineoffset = pin
+  rq.handleflags = GPIOHANDLE_REQUEST_INPUT
+  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
+  rq.label = label.encode('utf-8')[:31] + b'\0'
+
+  fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
+  fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
+  os.close(fd)
+  return int(rq.fd)

openpilot/common/gps.py

@@ -0,0 +1,8 @@
+from openpilot.common.params import Params
+
+
+def get_gps_location_service(params: Params) -> str:
+  if params.get_bool("UbloxAvailable"):
+    return "gpsLocationExternal"
+  else:
+    return "gpsLocation"

openpilot/common/hardware/__init__.py

@@ -0,0 +1,16 @@
+import os
+from typing import cast
+
+from openpilot.common.hardware.base import HardwareBase
+from openpilot.common.hardware.tici.hardware import Tici
+from openpilot.common.hardware.pc.hardware import Pc
+
+TICI = os.path.isfile('/TICI')
+AGNOS = os.path.isfile('/AGNOS')
+PC = not TICI
+
+
+if TICI:
+  HARDWARE = cast(HardwareBase, Tici())
+else:
+  HARDWARE = cast(HardwareBase, Pc())

openpilot/common/hardware/base.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <cstdlib>
+#include <fstream>
+#include <map>
+#include <string>
+
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
+
+// no-op base hw class
+class HardwareNone {
+public:
+  static std::string get_name() { return ""; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
+
+  static std::string get_serial() { return "cccccc"; }
+
+  static std::map<std::string, std::string> get_init_logs() {
+    return {};
+  }
+
+  static void set_ir_power(int percentage) {}
+
+  static bool PC() { return false; }
+};

openpilot/common/hardware/base.py

@@ -0,0 +1,161 @@
+import os
+from abc import abstractmethod, ABC
+from dataclasses import dataclass, fields
+
+from openpilot.cereal import log
+from openpilot.common.esim.base import LPABase
+
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+
+@dataclass
+class ThermalZone:
+  # a zone from /sys/class/thermal/thermal_zone*
+  name: str             # a.k.a type
+  scale: float = 1000.  # scale to get degrees in C
+  zone_number = -1
+
+  def read(self) -> float:
+    if self.zone_number < 0:
+      for n in os.listdir("/sys/devices/virtual/thermal"):
+        if not n.startswith("thermal_zone"):
+          continue
+        with open(os.path.join("/sys/devices/virtual/thermal", n, "type")) as f:
+          if f.read().strip() == self.name:
+            self.zone_number = int(n.removeprefix("thermal_zone"))
+            break
+
+    try:
+      with open(f"/sys/devices/virtual/thermal/thermal_zone{self.zone_number}/temp") as f:
+        return int(f.read()) / self.scale
+    except FileNotFoundError:
+      return 0
+
+@dataclass
+class ThermalConfig:
+  cpu: list[ThermalZone] | None = None
+  gpu: list[ThermalZone] | None = None
+  dsp: ThermalZone | None = None
+  pmic: list[ThermalZone] | None = None
+  memory: ThermalZone | None = None
+  intake: ThermalZone | None = None
+  exhaust: ThermalZone | None = None
+  gnss: ThermalZone | None = None
+  bottomSoc: ThermalZone | None = None
+
+  def get_msg(self):
+    ret = {}
+    for f in fields(ThermalConfig):
+      v = getattr(self, f.name)
+      if v is not None:
+        if isinstance(v, list):
+          ret[f.name + "TempC"] = [x.read() for x in v]
+        else:
+          ret[f.name + "TempC"] = v.read()
+    return ret
+
+class HardwareBase(ABC):
+  @staticmethod
+  def get_cmdline() -> dict[str, str]:
+    with open('/proc/cmdline') as f:
+      cmdline = f.read()
+    return {kv[0]: kv[1] for kv in [s.split('=') for s in cmdline.split(' ')] if len(kv) == 2}
+
+  @staticmethod
+  def read_param_file(path, parser, default=0):
+    try:
+      with open(path) as f:
+        return parser(f.read())
+    except Exception:
+      return default
+
+  def booted(self) -> bool:
+    return True
+
+  def reboot(self, reason=None):
+    print("REBOOT!")
+
+  def uninstall(self):
+    print("uninstall")
+
+  def get_os_version(self):
+    return None
+
+  @abstractmethod
+  def get_device_type(self):
+    pass
+
+  def get_imei(self) -> str:
+    return ""
+
+  def get_serial(self):
+    return ""
+
+  def get_network_info(self):
+    return None
+
+  def get_network_type(self):
+    return NetworkType.none
+
+  def get_sim_info(self):
+    return {
+      'sim_id': '',
+      'mcc_mnc': None,
+      'network_type': ["Unknown"],
+      'sim_state': ["ABSENT"],
+      'data_connected': False
+    }
+
+  def get_sim_lpa(self) -> LPABase:
+    raise NotImplementedError("SIM LPA not available")
+
+  def get_network_strength(self, network_type):
+    return NetworkStrength.unknown
+
+  def get_network_metered(self, network_type) -> bool:
+    return network_type not in (NetworkType.none, NetworkType.wifi, NetworkType.ethernet)
+
+  def get_current_power_draw(self):
+    return 0
+
+  def get_som_power_draw(self):
+    return 0
+
+  def shutdown(self):
+    print("SHUTDOWN!")
+
+  def get_thermal_config(self):
+    return ThermalConfig()
+
+  def set_display_power(self, on: bool):
+    pass
+
+  def set_screen_brightness(self, percentage):
+    pass
+
+  def get_screen_brightness(self):
+    return 0
+
+  def set_power_save(self, powersave_enabled):
+    pass
+
+  def get_gpu_usage_percent(self):
+    return 0
+
+  def get_modem_temperatures(self):
+    return []
+
+  def initialize_hardware(self):
+    pass
+
+  def reset_internal_panda(self):
+    pass
+
+  def recover_internal_panda(self):
+    pass
+
+  def get_modem_data_usage(self):
+    return -1, -1
+
+  def set_ir_power(self, percent: int):
+    pass

openpilot/common/hardware/hw.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <string>
+
+#include "common/hardware/base.h"
+#include "common/util.h"
+
+#if __TICI__
+#include "common/hardware/tici/hardware.h"
+#define Hardware HardwareTici
+#else
+#include "common/hardware/pc/hardware.h"
+#define Hardware HardwarePC
+#endif
+
+namespace Path {
+  inline std::string openpilot_prefix() {
+    return util::getenv("OPENPILOT_PREFIX", "");
+  }
+
+  inline std::string comma_home() {
+    return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix();
+  }
+
+  inline std::string log_root() {
+    if (const char *env = getenv("LOG_ROOT")) {
+      return env;
+    }
+    return Hardware::PC() ? Path::comma_home() + "/media/0/realdata" : "/data/media/0/realdata";
+  }
+
+  inline std::string params() {
+    return util::getenv("PARAMS_ROOT", Hardware::PC() ? (Path::comma_home() + "/params") : "/data/params");
+  }
+
+  inline std::string rsa_file() {
+    return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa";
+  }
+
+  inline std::string swaglog_ipc() {
+    return "ipc:///tmp/logmessage" + Path::openpilot_prefix();
+  }
+
+  inline std::string download_cache_root() {
+    if (const char *env = getenv("COMMA_CACHE")) {
+      return env;
+    }
+    return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/";
+  }
+
+ inline std::string shm_path() {
+    #ifdef __APPLE__
+     return"/tmp";
+    #else
+     return "/dev/shm";
+    #endif
+ }
+
+  inline std::string model_root() {
+    return Hardware::PC() ? Path::comma_home() + "/media/0/models" : "/data/media/0/models";
+  }
+}  // namespace Path

openpilot/common/hardware/hw.py

@@ -0,0 +1,97 @@
+import os
+import platform
+from pathlib import Path
+
+from openpilot.common.hardware import PC
+
+DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
+
+class Paths:
+  @staticmethod
+  def comma_home() -> str:
+    return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", ""))
+
+  @staticmethod
+  def log_root() -> str:
+    if os.environ.get('LOG_ROOT', False):
+      return os.environ['LOG_ROOT']
+    elif PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "realdata")
+    else:
+      return '/data/media/0/realdata/'
+
+  @staticmethod
+  def log_root_external() -> str:
+    return '/mnt/external_realdata/'
+
+  @staticmethod
+  def swaglog_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "log")
+    else:
+      return "/data/log/"
+
+  @staticmethod
+  def swaglog_ipc() -> str:
+    return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "")
+
+  @staticmethod
+  def download_cache_root() -> str:
+    if os.environ.get('COMMA_CACHE', False):
+      return os.environ['COMMA_CACHE'] + "/"
+    return DEFAULT_DOWNLOAD_CACHE_ROOT + os.environ.get("OPENPILOT_PREFIX", "") + "/"
+
+  @staticmethod
+  def persist_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "persist")
+    else:
+      return "/persist/"
+
+  @staticmethod
+  def stats_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "stats")
+    else:
+      return "/data/stats/"
+
+  @staticmethod
+  def stats_sp_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "stats")
+    else:
+      return "/data/stats_sp/"
+
+  @staticmethod
+  def config_root() -> str:
+    if PC:
+      return Paths.comma_home()
+    else:
+      return "/tmp/.comma"
+
+  @staticmethod
+  def shm_path() -> str:
+    if PC and platform.system() == "Darwin":
+      return "/tmp"  # This is not really shared memory on macOS, but it's the closest we can get
+    return "/dev/shm"
+
+  @staticmethod
+  def model_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "models")
+    else:
+      return "/data/media/0/models"
+
+  @staticmethod
+  def crash_log_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "community" / "crashes")
+    else:
+      return "/data/community/crashes"
+
+  @staticmethod
+  def mapd_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "osm")
+    else:
+      return "/data/media/0/osm"

openpilot/common/hardware/pc/hardware.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+
+#include "common/hardware/base.h"
+
+class HardwarePC : public HardwareNone {
+public:
+  static std::string get_name() { return "pc"; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
+  static bool PC() { return true; }
+};

openpilot/common/hardware/pc/hardware.py

@@ -0,0 +1,10 @@
+from openpilot.cereal import log
+from openpilot.common.hardware.base import HardwareBase
+
+class Pc(HardwareBase):
+  def get_device_type(self):
+    return "pc"
+
+  def get_network_type(self):
+    # some stuff is gated on wifi, so just assume for now
+    return log.DeviceState.NetworkType.wifi

openpilot/common/hardware/tici/__init__.py

@@ -0,0 +1 @@
+

openpilot/common/hardware/tici/agnos.json

@@ -0,0 +1,84 @@
+[
+  {
+    "name": "xbl",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl-e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb.img.xz",
+    "hash": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "hash_raw": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "size": 3282256,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "bea7f1a24428c3ededf672fa4fc78baf180cfbd8aafb77c974655b38517283e3"
+  },
+  {
+    "name": "xbl_config",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85.img.xz",
+    "hash": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "hash_raw": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "size": 98124,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "fb18cde08a98a168961ecd357e92474823046752b94e112f59fe51a6acd7197d"
+  },
+  {
+    "name": "abl",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c.img.xz",
+    "hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "hash_raw": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "size": 274432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c"
+  },
+  {
+    "name": "aop",
+    "url": "https://commadist.azureedge.net/agnosupdate/aop-78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1.img.xz",
+    "hash": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "hash_raw": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "size": 184364,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "6c9135446bd3fc075fcee59b887a12e49029ab1f98ed8d6d1e32c73569d47de3"
+  },
+  {
+    "name": "devcfg",
+    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc.img.xz",
+    "hash": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "hash_raw": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "size": 40336,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "2a67971602012c1b43544964709da13c322786b456a8e78568b117e8b1540ce3"
+  },
+  {
+    "name": "boot",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8.img.xz",
+    "hash": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "hash_raw": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "size": 17487872,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "edca8bee1531e66953d107eeceeed2dc7b3ca46417e49d55508f94e58bf95db8"
+  },
+  {
+    "name": "system",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img.xz",
+    "hash": "78acfe16a7b62a3a91fc7a81f40a693e4468cec1c69df7d0b1e550aacc646113",
+    "hash_raw": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+    "size": 4718592000,
+    "sparse": true,
+    "full_check": false,
+    "has_ab": true,
+    "ondevice_hash": "743142c5a898f27b2a1029cca42c8a5d5d1fc0096414422b850fe84c8d0b8342",
+    "alt": {
+      "hash": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img",
+      "size": 4718592000
+    }
+  }
+]

openpilot/common/hardware/tici/agnos.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+import hashlib
+import json
+import lzma
+import os
+import struct
+import subprocess
+import time
+from collections.abc import Generator
+
+import requests
+
+SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
+
+AGNOS_MANIFEST_FILE = "openpilot/system/hardware/tici/agnos.json"
+
+
+class StreamingDecompressor:
+  def __init__(self, url: str) -> None:
+    self.buf = b""
+
+    self.req = requests.get(url, stream=True, headers={'Accept-Encoding': None}, timeout=60)
+    self.it = self.req.iter_content(chunk_size=1024 * 1024)
+    self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
+    self.eof = False
+    self.sha256 = hashlib.sha256()
+
+  def read(self, length: int) -> bytes:
+    while len(self.buf) < length and not self.eof:
+      if self.decompressor.needs_input:
+        self.req.raise_for_status()
+
+        try:
+          compressed = next(self.it)
+        except StopIteration:
+          self.eof = True
+          break
+      else:
+        compressed = b''
+
+      self.buf += self.decompressor.decompress(compressed, max_length=length)
+
+      if self.decompressor.eof:
+        self.eof = True
+        break
+
+    result = self.buf[:length]
+    self.buf = self.buf[length:]
+
+    self.sha256.update(result)
+    return result
+
+
+def unsparsify(f: StreamingDecompressor) -> Generator[bytes, None, None]:
+  # https://source.android.com/devices/bootloader/images#sparse-format
+  magic = struct.unpack("I", f.read(4))[0]
+  assert(magic == 0xed26ff3a)
+
+  # Version
+  major = struct.unpack("H", f.read(2))[0]
+  minor = struct.unpack("H", f.read(2))[0]
+  assert(major == 1 and minor == 0)
+
+  f.read(2)  # file header size
+  f.read(2)  # chunk header size
+
+  block_sz = struct.unpack("I", f.read(4))[0]
+  f.read(4)  # total blocks
+  num_chunks = struct.unpack("I", f.read(4))[0]
+  f.read(4)  # crc checksum
+
+  for _ in range(num_chunks):
+    chunk_type, out_blocks = SPARSE_CHUNK_FMT.unpack(f.read(12))
+
+    if chunk_type == 0xcac1:  # Raw
+      # TODO: yield in smaller chunks. Yielding only block_sz is too slow. Largest observed data chunk is 252 MB.
+      yield f.read(out_blocks * block_sz)
+    elif chunk_type == 0xcac2:  # Fill
+      filler = f.read(4) * (block_sz // 4)
+      for _ in range(out_blocks):
+        yield filler
+    elif chunk_type == 0xcac3:  # Don't care
+      yield b""
+    else:
+      raise Exception("Unhandled sparse chunk type")
+
+
+# noop wrapper with same API as unsparsify() for non sparse images
+def noop(f: StreamingDecompressor) -> Generator[bytes, None, None]:
+  while len(chunk := f.read(1024 * 1024)) > 0:
+    yield chunk
+
+
+def get_target_slot_number() -> int:
+  current_slot = subprocess.check_output(["abctl", "--boot_slot"], encoding='utf-8').strip()
+  return 1 if current_slot == "_a" else 0
+
+
+def slot_number_to_suffix(slot_number: int) -> str:
+  assert slot_number in (0, 1)
+  return '_a' if slot_number == 0 else '_b'
+
+
+def get_partition_path(target_slot_number: int, partition: dict) -> str:
+  path = f"/dev/disk/by-partlabel/{partition['name']}"
+
+  if partition.get('has_ab', True):
+    path += slot_number_to_suffix(target_slot_number)
+
+  return path
+
+
+def get_raw_hash(path: str, partition_size: int) -> str:
+  raw_hash = hashlib.sha256()
+  pos, chunk_size = 0, 1024 * 1024
+
+  with open(path, 'rb+') as out:
+    while pos < partition_size:
+      n = min(chunk_size, partition_size - pos)
+      raw_hash.update(out.read(n))
+      pos += n
+
+  return raw_hash.hexdigest().lower()
+
+
+def verify_partition(target_slot_number: int, partition: dict[str, str | int], force_full_check: bool = False) -> bool:
+  full_check = partition['full_check'] or force_full_check
+  path = get_partition_path(target_slot_number, partition)
+
+  if not isinstance(partition['size'], int):
+    return False
+
+  partition_size: int = partition['size']
+
+  if not isinstance(partition['hash_raw'], str):
+    return False
+
+  partition_hash: str = partition['hash_raw']
+
+  if full_check:
+    return get_raw_hash(path, partition_size) == partition_hash.lower()
+  else:
+    with open(path, 'rb+') as out:
+      out.seek(partition_size)
+      return out.read(64) == partition_hash.lower().encode()
+
+
+def clear_partition_hash(target_slot_number: int, partition: dict) -> None:
+  path = get_partition_path(target_slot_number, partition)
+  with open(path, 'wb+') as out:
+    partition_size = partition['size']
+
+    out.seek(partition_size)
+    out.write(b"\x00" * 64)
+    os.sync()
+
+
+def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog):
+  path = get_partition_path(target_slot_number, partition)
+  downloader = StreamingDecompressor(partition['url'])
+
+  with open(path, 'wb+') as out:
+    # Flash partition
+    last_p = 0
+    raw_hash = hashlib.sha256()
+    f = unsparsify if partition['sparse'] else noop
+    for chunk in f(downloader):
+      raw_hash.update(chunk)
+      out.write(chunk)
+      p = int(out.tell() / partition['size'] * 100)
+      if p != last_p:
+        last_p = p
+        print(f"Installing {partition['name']}: {p}", flush=True)
+
+    if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
+      raise Exception(f"Raw hash mismatch '{raw_hash.hexdigest().lower()}'")
+
+    if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
+      raise Exception("Uncompressed hash mismatch")
+
+    if out.tell() != partition['size']:
+      raise Exception("Uncompressed size mismatch")
+
+    os.sync()
+
+
+def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False):
+  cloudlog.info(f"Downloading and writing {partition['name']}")
+
+  if verify_partition(target_slot_number, partition):
+    cloudlog.info(f"Already flashed {partition['name']}")
+    return
+
+  # Clear hash before flashing in case we get interrupted
+  full_check = partition['full_check']
+  if not full_check:
+    clear_partition_hash(target_slot_number, partition)
+
+  path = get_partition_path(target_slot_number, partition)
+
+  extract_compressed_image(target_slot_number, partition, cloudlog)
+
+  # Write hash after successful flash
+  if not full_check:
+    with open(path, 'wb+') as out:
+      out.seek(partition['size'])
+      out.write(partition['hash_raw'].lower().encode())
+
+
+def swap(manifest_path: str, target_slot_number: int, cloudlog) -> None:
+  update = json.load(open(manifest_path))
+  for partition in update:
+    if not partition.get('full_check', False):
+      clear_partition_hash(target_slot_number, partition)
+
+  while True:
+    out = subprocess.check_output(f"abctl --set_active {target_slot_number}", shell=True, stderr=subprocess.STDOUT, encoding='utf8')
+    if ("No such file or directory" not in out) and ("lun as boot lun" in out):
+      cloudlog.info(f"Swap successful {out}")
+      break
+    else:
+      cloudlog.error(f"Swap failed {out}")
+
+
+def flash_agnos_update(manifest_path: str, target_slot_number: int, cloudlog, standalone=False) -> None:
+  update = json.load(open(manifest_path))
+
+  cloudlog.info(f"Target slot {target_slot_number}")
+
+  # set target slot as unbootable
+  subprocess.run(f"abctl --set_unbootable {target_slot_number}", shell=True)
+
+  for partition in update:
+    success = False
+
+    for retries in range(10):
+      try:
+        flash_partition(target_slot_number, partition, cloudlog, standalone)
+        success = True
+        break
+
+      except requests.exceptions.RequestException:
+        cloudlog.exception("Failed")
+        cloudlog.info(f"Failed to download {partition['name']}, retrying ({retries})")
+        time.sleep(10)
+
+    if not success:
+      cloudlog.info(f"Failed to flash {partition['name']}, aborting")
+      raise Exception("Maximum retries exceeded")
+
+  cloudlog.info(f"AGNOS ready on slot {target_slot_number}")
+
+
+def verify_agnos_update(manifest_path: str, target_slot_number: int) -> bool:
+  update = json.load(open(manifest_path))
+  return all(verify_partition(target_slot_number, partition) for partition in update)
+
+
+if __name__ == "__main__":
+  import argparse
+  import logging
+
+  parser = argparse.ArgumentParser(description="Flash and verify AGNOS update",
+                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+  parser.add_argument("--verify", action="store_true", help="Verify and perform swap if update ready")
+  parser.add_argument("--swap", action="store_true", help="Verify and perform swap, downloads if necessary")
+  parser.add_argument("manifest", help="Manifest json")
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.INFO)
+
+  target_slot_number = get_target_slot_number()
+  if args.verify:
+    if verify_agnos_update(args.manifest, target_slot_number):
+      swap(args.manifest, target_slot_number, logging)
+      exit(0)
+    exit(1)
+  elif args.swap:
+    while not verify_agnos_update(args.manifest, target_slot_number):
+      logging.error("Verification failed. Flashing AGNOS")
+      flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)
+
+    logging.warning(f"Verification succeeded. Swapping to slot {target_slot_number}")
+    swap(args.manifest, target_slot_number, logging)
+  else:
+    flash_agnos_update(args.manifest, target_slot_number, logging, standalone=True)

openpilot/common/hardware/tici/all-partitions.json

@@ -0,0 +1,389 @@
+[
+  {
+    "name": "gpt_main_0",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_0-8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd.img.xz",
+    "hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "hash_raw": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "8928a31fd9ee20f8703649f89833eba9b55e84b6415e67799c777b163c95a0bd",
+    "gpt": {
+      "lun": 0,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_1",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_1-fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6.img.xz",
+    "hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "hash_raw": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "fe8ef7653db588d7420a625920ca06927dfcb0ed8aff3e3a1c74a52a24398ba6",
+    "gpt": {
+      "lun": 1,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_2",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_2-5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21.img.xz",
+    "hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "hash_raw": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "5ccfc7240c8cbfa2f1a018a2e376cf274a6baf858c9bfe71951d8e28cab53c21",
+    "gpt": {
+      "lun": 2,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_3",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_3-c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159.img.xz",
+    "hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "hash_raw": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "c707979fa21e89519328f4f30c2b21c9c453401ca8303f914c1873d410a95159",
+    "gpt": {
+      "lun": 3,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_4",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_4-e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e.img.xz",
+    "hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "hash_raw": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "e9405dcd785dbe79412184e1894a9c51ab7deb33bb612166c4c42a3d2bf42a0e",
+    "gpt": {
+      "lun": 4,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "gpt_main_5",
+    "url": "https://commadist.azureedge.net/agnosupdate/gpt_main_5-21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3.img.xz",
+    "hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "hash_raw": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "size": 24576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "21ae965f05b2fa8d02e04f1eb74718f9779864f6eacdeb859757d6435e8ccce3",
+    "gpt": {
+      "lun": 5,
+      "start_sector": 0,
+      "num_sectors": 6
+    }
+  },
+  {
+    "name": "persist",
+    "url": "https://commadist.azureedge.net/agnosupdate/persist-d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786.img.xz",
+    "hash": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786",
+    "hash_raw": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786",
+    "size": 4096,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "d6af4ec18df180c7417353b52a9e05e43a6480b29425f087874136436cefe786"
+  },
+  {
+    "name": "systemrw",
+    "url": "https://commadist.azureedge.net/agnosupdate/systemrw-8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e.img.xz",
+    "hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
+    "hash_raw": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e",
+    "size": 16777216,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "8ce150ca38ef64a0885fc2fe816e5b63bae8adb4df5d809c5b318e6996366c7e"
+  },
+  {
+    "name": "cache",
+    "url": "https://commadist.azureedge.net/agnosupdate/cache-ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4.img.xz",
+    "hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
+    "hash_raw": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4",
+    "size": 134217728,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "ebfbaaa2f96dc4e5fea4f126364e5bf5b3b44c12cbc753b62fdd8baab82f70b4"
+  },
+  {
+    "name": "xbl",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl-e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb.img.xz",
+    "hash": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "hash_raw": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "size": 3282256,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "bea7f1a24428c3ededf672fa4fc78baf180cfbd8aafb77c974655b38517283e3"
+  },
+  {
+    "name": "xbl_config",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85.img.xz",
+    "hash": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "hash_raw": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "size": 98124,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "fb18cde08a98a168961ecd357e92474823046752b94e112f59fe51a6acd7197d"
+  },
+  {
+    "name": "abl",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c.img.xz",
+    "hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "hash_raw": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "size": 274432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c"
+  },
+  {
+    "name": "aop",
+    "url": "https://commadist.azureedge.net/agnosupdate/aop-78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1.img.xz",
+    "hash": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "hash_raw": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "size": 184364,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "6c9135446bd3fc075fcee59b887a12e49029ab1f98ed8d6d1e32c73569d47de3"
+  },
+  {
+    "name": "bluetooth",
+    "url": "https://commadist.azureedge.net/agnosupdate/bluetooth-9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533.img.xz",
+    "hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
+    "hash_raw": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533",
+    "size": 1048576,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "9bb766d2d2ce0cc4491664b3010fe1ef62f8ffc1e362d55f78e48c4141f75533"
+  },
+  {
+    "name": "cmnlib64",
+    "url": "https://commadist.azureedge.net/agnosupdate/cmnlib64-1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3.img.xz",
+    "hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
+    "hash_raw": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "1a876bd151bb9635f18719c4a17f953079de6e11d3eaec800968fc75669e0dc3"
+  },
+  {
+    "name": "cmnlib",
+    "url": "https://commadist.azureedge.net/agnosupdate/cmnlib-63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82.img.xz",
+    "hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
+    "hash_raw": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "63df823e8a5fae01d66cb2b8c20f0d2ddb5c5f2425e5d0992a64676273ba1c82"
+  },
+  {
+    "name": "devcfg",
+    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc.img.xz",
+    "hash": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "hash_raw": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "size": 40336,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "2a67971602012c1b43544964709da13c322786b456a8e78568b117e8b1540ce3"
+  },
+  {
+    "name": "devinfo",
+    "url": "https://commadist.azureedge.net/agnosupdate/devinfo-143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3.img.xz",
+    "hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
+    "hash_raw": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3",
+    "size": 4096,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "143869c499a7e878fbeab756e9c53074195770cc41d6d0d10e45c043141389a3"
+  },
+  {
+    "name": "dsp",
+    "url": "https://commadist.azureedge.net/agnosupdate/dsp-4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248.img.xz",
+    "hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
+    "hash_raw": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248",
+    "size": 33554432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "4b15fbd2f45581f1553f33f01649e450b24aa19d5deff2ac7dcb16a534d9c248"
+  },
+  {
+    "name": "hyp",
+    "url": "https://commadist.azureedge.net/agnosupdate/hyp-ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927.img.xz",
+    "hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
+    "hash_raw": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "ff5ece6a4e3d2b4d898c77ffe193fc8bbc8acebe78263996ecf52373d8088927"
+  },
+  {
+    "name": "keymaster",
+    "url": "https://commadist.azureedge.net/agnosupdate/keymaster-5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04.img.xz",
+    "hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
+    "hash_raw": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04",
+    "size": 524288,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "5c968c76f29b9a4d66fbe57e639bac6b7a2c83b1758e25abbaf5d276b8a6af04"
+  },
+  {
+    "name": "limits",
+    "url": "https://commadist.azureedge.net/agnosupdate/limits-94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1.img.xz",
+    "hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
+    "hash_raw": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1",
+    "size": 4096,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "94951a0f7aa55fb6cb975535ce4ebbfe6d695f04cb5424677b01c10dfa2e94e1"
+  },
+  {
+    "name": "logfs",
+    "url": "https://commadist.azureedge.net/agnosupdate/logfs-b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220.img.xz",
+    "hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
+    "hash_raw": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220",
+    "size": 8388608,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "b8b5ac87f3d954404fc7ecbdd9ee3b5b0cf5691e5006e6ec55db4c899ff61220"
+  },
+  {
+    "name": "modem",
+    "url": "https://commadist.azureedge.net/agnosupdate/modem-a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994.img.xz",
+    "hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
+    "hash_raw": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994",
+    "size": 125829120,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "a3d014f0896d77a2df7e5a80a70f43a51a047b9d03cfc675b6f0e31a6ecc4994"
+  },
+  {
+    "name": "qupfw",
+    "url": "https://commadist.azureedge.net/agnosupdate/qupfw-64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a.img.xz",
+    "hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
+    "hash_raw": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a",
+    "size": 65536,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "64cc7c29d5d69b04267452b8b4ddba9f4809e68f476fc162ca283f58537afe4a"
+  },
+  {
+    "name": "splash",
+    "url": "https://commadist.azureedge.net/agnosupdate/splash-5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08.img.xz",
+    "hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
+    "hash_raw": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08",
+    "size": 34226176,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "5c61260048f22ede6e6343fabb27f6ff73f9271f4751a01aaf7abf097afc1f08"
+  },
+  {
+    "name": "storsec",
+    "url": "https://commadist.azureedge.net/agnosupdate/storsec-4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce.img.xz",
+    "hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
+    "hash_raw": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce",
+    "size": 131072,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "4494d86f68b125fbf2c004c824b1c6dbe71e61a65d2a1cc7db13c553edcb3fce"
+  },
+  {
+    "name": "tz",
+    "url": "https://commadist.azureedge.net/agnosupdate/tz-e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16.img.xz",
+    "hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
+    "hash_raw": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16",
+    "size": 2097152,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "e9443bf187641661bfa6c96702b9ab0156e72fb7482500f8799ba9ee2503cb16"
+  },
+  {
+    "name": "boot",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8.img.xz",
+    "hash": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "hash_raw": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "size": 17487872,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "edca8bee1531e66953d107eeceeed2dc7b3ca46417e49d55508f94e58bf95db8"
+  },
+  {
+    "name": "system",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img.xz",
+    "hash": "78acfe16a7b62a3a91fc7a81f40a693e4468cec1c69df7d0b1e550aacc646113",
+    "hash_raw": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+    "size": 4718592000,
+    "sparse": true,
+    "full_check": false,
+    "has_ab": true,
+    "ondevice_hash": "743142c5a898f27b2a1029cca42c8a5d5d1fc0096414422b850fe84c8d0b8342",
+    "alt": {
+      "hash": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img",
+      "size": 4718592000
+    }
+  },
+  {
+    "name": "userdata_90",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_90-14a3fc6e9bd148b9deebf6ae9df2f1b3b759e629b337e41b6895864cdd51f630.img.xz",
+    "hash": "52160dd01b30b3dc572226e8d549a034b03bc328e80f1f4cd6a857b6dd447687",
+    "hash_raw": "14a3fc6e9bd148b9deebf6ae9df2f1b3b759e629b337e41b6895864cdd51f630",
+    "size": 96636764160,
+    "sparse": true,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "3bbc052c7793087946b0cd668c1778f930084f6b00896aeebd193dfce53fa518"
+  },
+  {
+    "name": "userdata_89",
+    "url": "https://commadist.azureedge.net/agnosupdate/userdata_89-425c69d021f4ee2f767963bf7b991d1a492485fa465c5f5d001e1cf7de3d62a0.img.xz",
+    "hash": "02a8c5512754d7781d930d242be3fad01fbce652297c69dfe8722dc92b18dc09",
+    "hash_raw": "425c69d021f4ee2f767963bf7b991d1a492485fa465c5f5d001e1cf7de3d62a0",
+    "size": 95563022336,
+    "sparse": true,
+    "full_check": true,
+    "has_ab": false,
+    "ondevice_hash": "3667d500f91b08a4671d7c09623eb6ae8fc905d9273893b50689e214005c7fa6"
+  }
+]

openpilot/common/hardware/tici/amplifier.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import time
+from collections import namedtuple
+
+from openpilot.common.i2c import SMBus
+
+# https://datasheets.maximintegrated.com/en/ds/MAX98089.pdf
+
+AmpConfig = namedtuple('AmpConfig', ['name', 'value', 'register', 'offset', 'mask'])
+
+CONFIG = [
+  AmpConfig("MCLK prescaler", 0b01, 0x10, 4, 0b00110000),
+  AmpConfig("PM: enable speakers", 0b11, 0x4D, 4, 0b00110000),
+  AmpConfig("PM: enable DACs", 0b11, 0x4D, 0, 0b00000011),
+  AmpConfig("Enable PLL1", 0b1, 0x12, 7, 0b10000000),
+  AmpConfig("Enable PLL2", 0b1, 0x1A, 7, 0b10000000),
+  AmpConfig("DAI1: I2S mode", 0b00100, 0x14, 2, 0b01111100),
+  AmpConfig("DAI2: I2S mode", 0b00100, 0x1C, 2, 0b01111100),
+  AmpConfig("DAI1 Passband filtering: music mode", 0b1, 0x18, 7, 0b10000000),
+  AmpConfig("DAI1 voice mode gain (DV1G)", 0b00, 0x2F, 4, 0b00110000),
+  AmpConfig("DAI1 attenuation (DV1)", 0x0, 0x2F, 0, 0b00001111),
+  AmpConfig("DAI2 attenuation (DV2)", 0x0, 0x31, 0, 0b00001111),
+  AmpConfig("DAI2: DC blocking", 0b1, 0x20, 0, 0b00000001),
+  AmpConfig("DAI2: High sample rate", 0b0, 0x20, 3, 0b00001000),
+  AmpConfig("ALC enable", 0b1, 0x43, 7, 0b10000000),
+  AmpConfig("ALC/excursion limiter release time", 0b101, 0x43, 4, 0b01110000),
+  AmpConfig("ALC multiband enable", 0b1, 0x43, 3, 0b00001000),
+  AmpConfig("DAI1 EQ enable", 0b0, 0x49, 0, 0b00000001),
+  AmpConfig("DAI2 EQ clip detection disabled", 0b1, 0x32, 4, 0b00010000),
+  AmpConfig("DAI2 EQ attenuation", 0x5, 0x32, 0, 0b00001111),
+  AmpConfig("Excursion limiter upper corner freq", 0b100, 0x41, 4, 0b01110000),
+  AmpConfig("Excursion limiter lower corner freq", 0b00, 0x41, 0, 0b00000011),
+  AmpConfig("Excursion limiter threshold", 0b000, 0x42, 0, 0b00001111),
+  AmpConfig("Distortion limit (THDCLP)", 0x6, 0x46, 4, 0b11110000),
+  AmpConfig("Distortion limiter release time constant", 0b0, 0x46, 0, 0b00000001),
+  AmpConfig("Right DAC input mixer: DAI1 left", 0b0, 0x22, 3, 0b00001000),
+  AmpConfig("Right DAC input mixer: DAI1 right", 0b0, 0x22, 2, 0b00000100),
+  AmpConfig("Right DAC input mixer: DAI2 left", 0b1, 0x22, 1, 0b00000010),
+  AmpConfig("Right DAC input mixer: DAI2 right", 0b0, 0x22, 0, 0b00000001),
+  AmpConfig("DAI1 audio port selector", 0b10, 0x16, 6, 0b11000000),
+  AmpConfig("DAI2 audio port selector", 0b01, 0x1E, 6, 0b11000000),
+  AmpConfig("Enable left digital microphone", 0b1, 0x48, 5, 0b00100000),
+  AmpConfig("Enable right digital microphone", 0b1, 0x48, 4, 0b00010000),
+  AmpConfig("Enhanced volume smoothing disabled", 0b0, 0x49, 7, 0b10000000),
+  AmpConfig("Volume adjustment smoothing disabled", 0b0, 0x49, 6, 0b01000000),
+  AmpConfig("Zero-crossing detection disabled", 0b0, 0x49, 5, 0b00100000),
+
+  AmpConfig("Left speaker output from left DAC", 0b1, 0x2B, 0, 0b11111111),
+  AmpConfig("Right speaker output from right DAC", 0b1, 0x2C, 0, 0b11111111),
+  AmpConfig("Left Speaker Mixer Gain", 0b00, 0x2D, 0, 0b00000011),
+  AmpConfig("Right Speaker Mixer Gain", 0b00, 0x2D, 2, 0b00001100),
+  AmpConfig("Left speaker output volume", 0x17, 0x3D, 0, 0b00011111),
+  AmpConfig("Right speaker output volume", 0x17, 0x3E, 0, 0b00011111),
+
+  AmpConfig("DAI2 EQ enable", 0b0, 0x49, 1, 0b00000010),
+  AmpConfig("DAI2: DC blocking", 0b0, 0x20, 0, 0b00000001),
+  AmpConfig("ALC enable", 0b0, 0x43, 7, 0b10000000),
+  AmpConfig("DAI2 EQ attenuation", 0x2, 0x32, 0, 0b00001111),
+  AmpConfig("Excursion limiter upper corner freq", 0b001, 0x41, 4, 0b01110000),
+  AmpConfig("Excursion limiter threshold", 0b100, 0x42, 0, 0b00001111),
+  AmpConfig("Distortion limit (THDCLP)", 0x0, 0x46, 4, 0b11110000),
+  AmpConfig("Distortion limiter release time constant", 0b1, 0x46, 0, 0b00000001),
+  AmpConfig("Left DAC input mixer: DAI1 left", 0b0, 0x22, 7, 0b10000000),
+  AmpConfig("Left DAC input mixer: DAI1 right", 0b0, 0x22, 6, 0b01000000),
+  AmpConfig("Left DAC input mixer: DAI2 left", 0b1, 0x22, 5, 0b00100000),
+  AmpConfig("Left DAC input mixer: DAI2 right", 0b0, 0x22, 4, 0b00010000),
+  AmpConfig("Right DAC input mixer: DAI2 left", 0b0, 0x22, 1, 0b00000010),
+  AmpConfig("Right DAC input mixer: DAI2 right", 0b1, 0x22, 0, 0b00000001),
+  AmpConfig("Volume adjustment smoothing disabled", 0b1, 0x49, 6, 0b01000000),
+]
+
+class Amplifier:
+  AMP_I2C_BUS = 0
+  AMP_ADDRESS = 0x10
+
+  def __init__(self, debug=False):
+    self.debug = debug
+
+  def _get_shutdown_config(self, amp_disabled: bool) -> AmpConfig:
+    return AmpConfig("Global shutdown", 0b0 if amp_disabled else 0b1, 0x51, 7, 0b10000000)
+
+  def _set_configs(self, configs: list[AmpConfig]) -> None:
+    with SMBus(self.AMP_I2C_BUS) as bus:
+      for config in configs:
+        if self.debug:
+          print(f"Setting \"{config.name}\" to {config.value}:")
+
+        old_value = bus.read_byte_data(self.AMP_ADDRESS, config.register, force=True)
+        new_value = (old_value & (~config.mask)) | ((config.value << config.offset) & config.mask)
+        bus.write_byte_data(self.AMP_ADDRESS, config.register, new_value, force=True)
+
+        if self.debug:
+          print(f"  Changed {hex(config.register)}: {hex(old_value)} -> {hex(new_value)}")
+
+  def set_configs(self, configs: list[AmpConfig]) -> bool:
+    # retry in case panda is using the amp
+    tries = 15
+    backoff = 0.
+    for i in range(tries):
+      try:
+        self._set_configs(configs)
+        return True
+      except OSError:
+        backoff += 0.1
+        time.sleep(backoff)
+        print(f"Failed to set amp config, {tries - i - 1} retries left")
+    return False
+
+  def set_global_shutdown(self, amp_disabled: bool) -> bool:
+    return self.set_configs([self._get_shutdown_config(amp_disabled), ])
+
+  def initialize_configuration(self) -> bool:
+    cfgs = [
+      self._get_shutdown_config(True),
+      *CONFIG,
+      self._get_shutdown_config(False),
+    ]
+    return self.set_configs(cfgs)
+
+
+if __name__ == "__main__":
+  amp = Amplifier()
+  amp.initialize_configuration()

openpilot/common/hardware/tici/hardware.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#include <cassert>
+#include <fstream>
+#include <map>
+#include <string>
+#include <algorithm>  // for std::clamp
+
+#include "common/util.h"
+#include "common/hardware/base.h"
+
+class HardwareTici : public HardwareNone {
+public:
+  static std::string get_name() {
+    static const std::string name = []() {
+      std::string model = util::read_file("/sys/firmware/devicetree/base/model");
+      return util::strip(model.substr(std::string("comma ").size()));
+    }();
+    return name;
+  }
+
+  static cereal::InitData::DeviceType get_device_type() {
+    static const std::map<std::string, cereal::InitData::DeviceType> device_map = {
+      {"tizi", cereal::InitData::DeviceType::TIZI},
+      {"mici", cereal::InitData::DeviceType::MICI}
+    };
+    static const auto it = device_map.find(get_name());
+    assert(it != device_map.end());
+    return it->second;
+  }
+
+  static std::string get_serial() {
+    static std::string serial("");
+    if (serial.empty()) {
+      std::ifstream stream("/proc/cmdline");
+      std::string cmdline;
+      std::getline(stream, cmdline);
+
+      auto start = cmdline.find("serialno=");
+      if (start == std::string::npos) {
+        serial = "cccccc";
+      } else {
+        auto end = cmdline.find(" ", start + 9);
+        serial = cmdline.substr(start + 9, end - start - 9);
+      }
+    }
+    return serial;
+  }
+
+  static void set_ir_power(int percent) {
+    auto device = get_device_type();
+    if (device == cereal::InitData::DeviceType::TIZI) {
+      return;
+    }
+
+    int value = util::map_val(std::clamp(percent, 0, 100), 0, 100, 0, 300);
+    std::ofstream("/sys/class/leds/led:switch_2/brightness") << 0 << "\n";
+    std::ofstream("/sys/class/leds/led:torch_2/brightness") << value << "\n";
+    std::ofstream("/sys/class/leds/led:switch_2/brightness") << value << "\n";
+  }
+
+  static std::map<std::string, std::string> get_init_logs() {
+    std::map<std::string, std::string> ret = {
+      {"/BUILD", util::read_file("/BUILD")},
+      {"lsblk", util::check_output("lsblk -o NAME,SIZE,STATE,VENDOR,MODEL,REV,SERIAL")},
+      {"SOM ID", util::read_file("/sys/devices/platform/vendor/vendor:gpio-som-id/som_id")},
+    };
+
+    std::string bs = util::check_output("abctl --boot_slot");
+    ret["boot slot"] = bs.substr(0, bs.find_first_of("\n"));
+
+    std::string temp = util::read_file("/dev/disk/by-partlabel/ssd");
+    temp.erase(temp.find_last_not_of(std::string("\0\r\n", 3))+1);
+    ret["boot temp"] = temp;
+
+    // TODO: log something from system and boot
+    for (std::string part : {"xbl", "abl", "aop", "devcfg", "xbl_config"}) {
+      for (std::string slot : {"a", "b"}) {
+        std::string partition = part + "_" + slot;
+        std::string hash = util::check_output("sha256sum /dev/disk/by-partlabel/" + partition);
+        ret[partition] = hash.substr(0, hash.find_first_of(" "));
+      }
+    }
+
+    return ret;
+  }
+};

openpilot/common/hardware/tici/hardware.py

@@ -0,0 +1,419 @@
+import configparser
+import json
+import os
+import socket
+import subprocess
+import time
+from functools import cached_property, lru_cache
+from pathlib import Path
+
+from openpilot.cereal import log
+from openpilot.common.utils import sudo_read, sudo_write
+from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
+from openpilot.common.esim.base import LPABase
+from openpilot.common.hardware.base import HardwareBase, ThermalConfig, ThermalZone
+from openpilot.common.esim.lpa import TiciLPA
+from openpilot.common.hardware.tici.pins import GPIO
+from openpilot.common.hardware.tici.amplifier import Amplifier
+
+MODEM_STATE_PATH = "/dev/shm/modem"
+
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+
+
+def affine_irq(val, action):
+  irqs = get_irqs_for_action(action)
+  if len(irqs) == 0:
+    print(f"No IRQs found for '{action}'")
+    return
+
+  for i in irqs:
+    sudo_write(str(val), f"/proc/irq/{i}/smp_affinity_list")
+
+@lru_cache
+def get_device_type():
+  # lru_cache and cache can cause memory leaks when used in classes
+  with open("/sys/firmware/devicetree/base/model") as f:
+    model = f.read().strip('\x00')
+  return model.split('comma ')[-1]
+
+def wpa_supplicant_cmd(cmd: str, timeout: float = 0.2) -> dict[str, str]:
+  with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+    sock.settimeout(timeout)
+    sock.bind(f"\0openpilot-wpa-{os.getpid()}-{time.monotonic_ns()}")
+    sock.connect("/run/wpa_supplicant/wlan0")
+    sock.send(cmd.encode())
+
+    while True:
+      out = sock.recv(8192).decode("utf-8", "replace")
+      if out.startswith("<"):
+        continue
+      if out.startswith("FAIL"):
+        return {}
+      return dict(l.split("=", 1) for l in out.splitlines() if "=" in l)
+
+def get_default_route_iface():
+  with open("/proc/net/route") as f:
+    routes = [(int(route[6]), route[0]) for line in f.readlines()[1:] if (route := line.split())[1] == "00000000" and int(route[3], 16) & 0x1]
+  return min(routes)[1] if routes else None
+
+class Tici(HardwareBase):
+  @cached_property
+  def amplifier(self):
+    if self.get_device_type() == "mici":
+      return None
+    return Amplifier()
+
+  def get_modem_state(self) -> dict:
+    try:
+      with open(MODEM_STATE_PATH) as f:
+        return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+      return {}
+
+  def get_os_version(self):
+    with open("/VERSION") as f:
+      return f.read().strip()
+
+  def get_device_type(self):
+    return get_device_type()
+
+  def reboot(self, reason=None):
+    subprocess.check_output(["sudo", "reboot"])
+
+  def uninstall(self):
+    Path("/data/__system_reset__").touch()
+    os.sync()
+    self.reboot()
+
+  def get_serial(self):
+    return self.get_cmdline()['androidboot.serialno']
+
+  def get_voltage(self):
+    with open("/sys/class/hwmon/hwmon1/in1_input") as f:
+      return int(f.read())
+
+  def get_current(self):
+    with open("/sys/class/hwmon/hwmon1/curr1_input") as f:
+      return int(f.read())
+
+  def set_ir_power(self, percent: int):
+    if self.get_device_type() == "tizi":
+      return
+
+    value = int((percent / 100) * 300)
+    with open("/sys/class/leds/led:switch_2/brightness", "w") as f:
+      f.write("0\n")
+    with open("/sys/class/leds/led:torch_2/brightness", "w") as f:
+      f.write(f"{value}\n")
+    with open("/sys/class/leds/led:switch_2/brightness", "w") as f:
+      f.write(f"{value}\n")
+
+  def get_network_type(self):
+    try:
+      if (iface := get_default_route_iface()):
+        if iface.startswith('wlan'):
+          return NetworkType.wifi
+        if iface.startswith('eth'):
+          return NetworkType.ethernet
+    except Exception:
+      pass
+
+    ms = self.get_modem_state()
+    if ms.get('connected'):
+      nt = ms.get('network_type', '')
+      if nt == 'nr':
+        return NetworkType.cell5G
+      elif nt == 'lte':
+        return NetworkType.cell4G
+      elif nt in ('utran', 'umts'):
+        return NetworkType.cell3G
+      elif nt == 'gsm':
+        return NetworkType.cell2G
+    return NetworkType.none
+
+  def get_sim_info(self):
+    ms = self.get_modem_state()
+    sim_id = ms.get('iccid', '')
+    return {
+      'sim_id': sim_id,
+      'mcc_mnc': ms.get('mcc_mnc') or None,
+      'network_type': ["Unknown"],
+      'sim_state': ["ABSENT"] if not sim_id else ["READY"],
+      'data_connected': ms.get('connected', False),
+    }
+
+  def get_sim_lpa(self) -> LPABase:
+    return TiciLPA()
+
+  def get_imei(self):
+    return self.get_modem_state().get('imei', '')
+
+  def get_network_info(self):
+    if self.get_device_type() == "mici":
+      return None
+
+    ms = self.get_modem_state()
+    return {
+      'technology': ms.get('network_type', '').upper() if ms.get('network_type') else '',
+      'operator': ms.get('operator', ''),
+      'band': ms.get('band', ''),
+      'channel': ms.get('channel', 0),
+      'extra': ms.get('extra', ''),
+      'state': ms.get('state', 'UNKNOWN'),
+    }
+
+  def parse_strength(self, percentage):
+    if percentage < 25:
+      return NetworkStrength.poor
+    elif percentage < 50:
+      return NetworkStrength.moderate
+    elif percentage < 75:
+      return NetworkStrength.good
+    else:
+      return NetworkStrength.great
+
+  def get_network_strength(self, network_type):
+    network_strength = NetworkStrength.unknown
+
+    try:
+      if network_type == NetworkType.none:
+        pass
+      elif network_type == NetworkType.ethernet:
+        network_strength = NetworkStrength.great
+      elif network_type == NetworkType.wifi:
+        rssi = wpa_supplicant_cmd("SIGNAL_POLL").get("RSSI")
+        if rssi is not None:
+          dbm = int(rssi)
+          if -100 < dbm <= 0:
+            network_strength = self.parse_strength(120 + max(-100, min(-20, dbm)))
+      else:  # Cellular
+        network_strength = self.parse_strength(self.get_modem_state().get('signal_quality', 0))
+    except Exception:
+      pass
+
+    return network_strength
+
+  def get_network_metered(self, network_type) -> bool:
+    if network_type in (NetworkType.cell2G, NetworkType.cell3G, NetworkType.cell4G, NetworkType.cell5G):
+      from openpilot.common.params import Params
+      return Params().get_bool("GsmMetered")
+    try:
+      if network_type == NetworkType.wifi:
+        ssid = wpa_supplicant_cmd("STATUS").get("ssid", "")
+        if ssid:
+          # wpa_supplicant escapes non-printable bytes as \xNN; NM keyfile stores ASCII SSIDs as a literal and others as a byte;byte; list
+          ssid_bytes = ssid.encode().decode('unicode_escape').encode('latin-1')
+          ssid_keyfile_list = ';'.join(str(b) for b in ssid_bytes) + ';'
+
+          nm_dirs = ("/run/NetworkManager/system-connections", "/data/etc/NetworkManager/system-connections")
+          for fpath in (p for d in nm_dirs for p in Path(d).glob("*.nmconnection")):
+            raw = sudo_read(str(fpath))
+            if not raw:
+              continue
+            cp = configparser.ConfigParser(interpolation=None)
+            try:
+              cp.read_string(raw)
+              keyfile_ssid = cp.get("wifi", "ssid", fallback="")
+              if keyfile_ssid != ssid and keyfile_ssid != ssid_keyfile_list:
+                continue
+              metered = cp.getint("connection", "metered", fallback=0)
+            except (configparser.Error, ValueError):
+              continue
+            if metered == 1:  # NM_METERED_YES
+              return True
+            if metered == 2:  # NM_METERED_NO
+              return False
+            break
+    except Exception:
+      pass
+
+    return super().get_network_metered(network_type)
+
+  def get_modem_temperatures(self):
+    return self.get_modem_state().get('temperatures', [])
+
+  def get_current_power_draw(self):
+    return (self.read_param_file("/sys/class/hwmon/hwmon1/power1_input", int) / 1e6)
+
+  def get_som_power_draw(self):
+    return (self.read_param_file("/sys/class/power_supply/bms/voltage_now", int) * self.read_param_file("/sys/class/power_supply/bms/current_now", int) / 1e12)
+
+  def shutdown(self):
+    subprocess.run("sudo poweroff", shell=True)
+
+  def get_thermal_config(self):
+    intake, exhaust, gnss, bottomSoc = None, None, None, None
+    if self.get_device_type() == "mici":
+      gnss = ThermalZone("gnss")
+      intake = ThermalZone("intake")
+      exhaust = ThermalZone("exhaust")
+      bottomSoc = ThermalZone("bottom_soc")
+    return ThermalConfig(cpu=[ThermalZone(f"cpu{i}-silver-usr") for i in range(4)] +
+                             [ThermalZone(f"cpu{i}-gold-usr") for i in range(4)],
+                         gpu=[ThermalZone("gpu0-usr"), ThermalZone("gpu1-usr")],
+                         dsp=ThermalZone("compute-hvx-usr"),
+                         memory=ThermalZone("ddr-usr"),
+                         pmic=[ThermalZone("pm8998_tz"), ThermalZone("pm8005_tz")],
+                         intake=intake,
+                         exhaust=exhaust,
+                         gnss=gnss,
+                         bottomSoc=bottomSoc)
+
+  def set_display_power(self, on):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/bl_power", "w") as f:
+        f.write("0" if on else "4")
+    except Exception:
+      pass
+
+  def set_screen_brightness(self, percentage):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
+        max_brightness = float(f.read().strip())
+
+      val = int(percentage * (max_brightness / 100.))
+      with open("/sys/class/backlight/panel0-backlight/brightness", "w") as f:
+        f.write(str(val))
+    except Exception:
+      pass
+
+  def get_screen_brightness(self):
+    try:
+      with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
+        max_brightness = float(f.read().strip())
+
+      with open("/sys/class/backlight/panel0-backlight/brightness") as f:
+        return int(float(f.read()) / (max_brightness / 100.))
+    except Exception:
+      return 0
+
+  def set_power_save(self, powersave_enabled):
+    # amplifier, 100mW at idle
+    if self.amplifier is not None:
+      self.amplifier.set_global_shutdown(amp_disabled=powersave_enabled)
+      if not powersave_enabled:
+        self.amplifier.initialize_configuration()
+
+    # *** CPU config ***
+
+    # offline big cluster
+    for i in range(4, 8):
+      val = '0' if powersave_enabled else '1'
+      sudo_write(val, f'/sys/devices/system/cpu/cpu{i}/online')
+
+    for n in ('0', '4'):
+      if powersave_enabled and n == '4':
+        continue
+      gov = 'ondemand' if powersave_enabled else 'performance'
+      sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
+      if not powersave_enabled:
+        # cap max core freq to 1689 Mhz
+        sudo_write('1689600', f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_max_freq')
+
+    # *** IRQ config ***
+
+    # GPU, modeld core
+    affine_irq(7, "kgsl-3d0")
+
+    # camerad core
+    camera_irqs = ("a5", "cci", "cpas_camnoc", "cpas-cdm", "csid", "ife", "csid-lite", "ife-lite")
+    for n in camera_irqs:
+      affine_irq(6, n)
+
+  def get_gpu_usage_percent(self):
+    try:
+      with open('/sys/class/kgsl/kgsl-3d0/gpubusy') as f:
+        used, total = f.read().strip().split()
+      return 100.0 * int(used) / int(total)
+    except Exception:
+      return 0
+
+  def initialize_hardware(self):
+    if self.amplifier is not None:
+      self.amplifier.initialize_configuration()
+
+    # Allow hardwared to write engagement status to kmsg
+    subprocess.run("sudo chmod a+w /dev/kmsg", shell=True)
+
+    # Ensure fan gpio is enabled so fan runs until shutdown, also turned on at boot by the ABL
+    gpio_init(GPIO.SOM_ST_IO, True)
+    gpio_set(GPIO.SOM_ST_IO, 1)
+
+    # *** IRQ config ***
+
+    # mask off big cluster from default affinity
+    sudo_write("f", "/proc/irq/default_smp_affinity")
+
+    # move these off the default core
+    affine_irq(1, "msm_vidc")  # encoders
+    affine_irq(1, "i2c_geni")  # sensors
+
+    # *** GPU config ***
+    # https://github.com/commaai/agnos-kernel-sdm845/blob/master/arch/arm64/boot/dts/qcom/sdm845-gpu.dtsi#L216
+    affine_irq(5, "fts_ts")    # touch
+    affine_irq(5, "msm_drm")   # display
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/min_pwrlevel")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/max_pwrlevel")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_bus_on")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_clk_on")
+    sudo_write("1", "/sys/class/kgsl/kgsl-3d0/force_rail_on")
+    sudo_write("1000", "/sys/class/kgsl/kgsl-3d0/idle_timer")
+    sudo_write("performance", "/sys/class/kgsl/kgsl-3d0/devfreq/governor")
+    sudo_write("710", "/sys/class/kgsl/kgsl-3d0/max_clock_mhz")
+
+    # setup governors
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,cpubw/governor")
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu0/governor")
+    sudo_write("performance", "/sys/class/devfreq/soc:qcom,memlat-cpu4/governor")
+
+    # *** VIDC (encoder) config ***
+    sudo_write("N", "/sys/kernel/debug/msm_vidc/clock_scaling")
+    sudo_write("Y", "/sys/kernel/debug/msm_vidc/disable_thermal_mitigation")
+
+    # pandad core
+    affine_irq(3, "spi_geni")         # SPI
+    try:
+      pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
+      subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
+      subprocess.call(["sudo", "taskset", "-pc", "3", pid])
+    except subprocess.CalledProcessError as e:
+      print(str(e))
+
+  def get_modem_data_usage(self):
+    ms = self.get_modem_state()
+    return ms.get('tx_bytes', -1), ms.get('rx_bytes', -1)
+
+  def reset_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_init(GPIO.STM_BOOT0, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    gpio_set(GPIO.STM_BOOT0, 0)
+    time.sleep(0.01)
+    gpio_set(GPIO.STM_RST_N, 0)
+
+  def recover_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_init(GPIO.STM_BOOT0, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    gpio_set(GPIO.STM_BOOT0, 1)
+    time.sleep(0.01)
+    gpio_set(GPIO.STM_RST_N, 0)
+    time.sleep(0.01)
+    gpio_set(GPIO.STM_BOOT0, 0)
+
+  def booted(self):
+    # this normally boots within 8s, but on rare occasions takes 30+s
+    encoder_state = sudo_read("/sys/kernel/debug/msm_vidc/core0/info")
+    if "Core state: 0" in encoder_state and (time.monotonic() < 60*2):
+      return False
+    return True
+
+if __name__ == "__main__":
+  t = Tici()
+  t.initialize_hardware()
+  t.set_power_save(False)
+  print(t.get_sim_info())

openpilot/common/hardware/tici/id_rsa

@@ -0,0 +1,28 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
+Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
+t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
+Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
+acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
+1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
+rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
+x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
+gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
+Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
+LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
+s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
+Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
+6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
+uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
+0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
+FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
+ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
+4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
+nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
+VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
+s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
+jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
+b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
+RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
+6co17RFaAZHwGfCFFjO76Q==
+-----END RSA PRIVATE KEY-----

openpilot/common/hardware/tici/modem.py

@@ -0,0 +1,592 @@
+#!/usr/bin/env python3
+import fcntl
+import json
+import logging
+import os
+import signal
+import subprocess
+import tempfile
+import time
+
+from ipaddress import IPv4Address, AddressValueError
+
+from enum import Enum
+
+from openpilot.common.serial import Serial
+
+logging.basicConfig(
+  level=logging.INFO,
+  format="%(asctime)s.%(msecs)03d %(levelname)-7s modem: %(message)s",
+  datefmt="%H:%M:%S",
+)
+
+AT_PORT = "/dev/modem_at0"
+PPP_PORT = "/dev/modem_at1"
+STATE_PATH = "/dev/shm/modem"
+AT_LOCK = "/dev/shm/modem.lock"  # shared with LPA
+AT_INIT = [
+  "ATE0",       # disable command echo
+  "ATV1",       # verbose result codes (CONNECT/BUSY/NO CARRIER, not numeric)
+  "AT+CMEE=1",  # numeric +CME ERROR codes on failures (per 3GPP 27.007)
+  "ATX4",       # extended result codes: busy + dial tone detection, line speed in CONNECT
+  "AT&C1",      # DCD pin follows carrier state (V.250 default)
+  "AT+CREG=2",  # registration URCs include location info
+  "AT+CGREG=2", # GPRS registration URCs include location info
+]
+CREG = {0: "not_registered", 1: "home", 2: "searching", 3: "denied", 4: "unknown", 5: "roaming"}
+# 3GPP TS 27.007 +COPS <AcT> -> network type
+NETWORK_TYPE = {0: "gsm", 1: "gsm", 3: "gsm", 8: "gsm",
+                2: "utran", 4: "utran", 5: "utran", 6: "utran",
+                7: "lte", 9: "lte", 10: "lte",
+                11: "nr", 12: "nr", 13: "nr"}
+
+DIAL_CID = 1
+WEBBING_ICCID_PREFIX = "8985235"
+
+PPPD_CMD = [
+  "sudo", "pppd", PPP_PORT, "460800", "noauth", "nodetach", "noipdefault", "usepeerdns",
+  "nodefaultroute", "connect",
+  "/usr/sbin/chat -v ABORT 'NO CARRIER' ABORT 'NO DIALTONE' ABORT 'BUSY' " +
+  f"ABORT 'NO ANSWER' ABORT 'ERROR' TIMEOUT 5 '' AT OK ATD*99***{DIAL_CID}# CONNECT ''",
+  "lcp-echo-interval", "30", "lcp-echo-failure", "4", "mtu", "1500", "mru", "1500",
+  "novj", "novjccomp", "ipcp-accept-local", "ipcp-accept-remote", "nomagic",
+  "user", '""', "password", '""',
+]
+INITIAL_STATE = {
+  "seconds_since_boot": 0,
+  "state": "INITIALIZING",
+  "connected": False, "ip_address": "",
+  "iccid": "", "mcc_mnc": "", "imei": "", "modem_version": "",
+  "signal_strength": 0, "signal_quality": 0,
+  "network_type": "unknown", "operator": "", "band": "", "channel": 0,
+  "registration": "unknown", "temperatures": [], "extra": "",
+  "tx_bytes": 0, "rx_bytes": 0,
+}
+
+
+class State(Enum):
+  INITIALIZING = "INITIALIZING"
+  SEARCHING = "SEARCHING"
+  CONNECTING = "CONNECTING"
+  CONNECTED = "CONNECTED"
+  DISCONNECTING = "DISCONNECTING"
+
+
+STATE_WAIT = 1.0  # seconds to wait after each state handler returns
+
+
+class PPPSession:
+  """Owns pppd lifecycle, fail tracking, and PPP routing."""
+  MAX_FAILS = 3
+
+  def __init__(self):
+    self._proc: subprocess.Popen | None = None
+    self._fails = 0
+    self._peer = ""
+
+  def start(self):
+    self._proc = subprocess.Popen(PPPD_CMD, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    self._peer = ""
+    logging.info(f"PPP dialing CID {DIAL_CID}")
+
+  def kill(self):
+    subprocess.run(["sudo", "killall", "-9", "pppd"], capture_output=True)
+    self._peer = ""
+
+  @staticmethod
+  def reset_data_port():
+    """Drop DTR on PPP_PORT so the modem terminates any stuck PPP session."""
+    try:
+      with Serial(PPP_PORT, baudrate=460800, timeout=1) as s:
+        s.dtr = False
+        time.sleep(0.2)
+        s.dtr = True
+    except Exception as e:
+      logging.warning(f"data port reset failed: {e}")
+
+  def has_exited(self) -> bool:
+    return self._proc is not None and self._proc.poll() is not None
+
+  def reset_fail_counter(self):
+    self._fails = 0
+
+  def record_fail(self) -> bool:
+    """Bump fail counter; return True if at the give-up limit."""
+    self._fails += 1
+    return self._fails >= self.MAX_FAILS
+
+  @property
+  def fails(self) -> int:
+    return self._fails
+
+  def maybe_install_routes(self, ip: str, peer: str) -> bool:
+    """Install routes if peer changed; kill the session on failure so the state machine reconnects."""
+    if not peer or peer == self._peer:
+      return False
+    try:
+      IPv4Address(ip)
+      IPv4Address(peer)
+    except AddressValueError:
+      logging.warning(f"refusing route install with non-IPv4 ip={ip!r} peer={peer!r}")
+      self.kill()
+      return False
+    self.cleanup_routes()
+    cmds = [
+      ["sudo", "ip", "route", "add", "default", "via", peer, "dev", "ppp0", "metric", "1000"],
+      ["sudo", "ip", "route", "add", "default", "via", peer, "dev", "ppp0", "table", "1000"],
+      ["sudo", "ip", "rule", "add", "from", ip, "table", "1000"],
+    ]
+    for cmd in cmds:
+      r = subprocess.run(cmd, capture_output=True, text=True)
+      if r.returncode != 0:
+        logging.warning(f"route install failed ({' '.join(cmd[1:])}): {r.stderr.strip()}")
+        self.cleanup_routes()
+        self.kill()
+        return False
+    logging.info(f"route set up for {ip} via {peer}")
+    self._peer = peer
+    return True
+
+  def maybe_install_dns(self, dns_servers: list[str]) -> bool:
+    """Register DNS servers with systemd-resolved; kill the session on failure to force a retry."""
+    if not dns_servers:
+      return False
+    for cmd in (["sudo", "resolvectl", "dns", "ppp0", *dns_servers],
+                ["sudo", "resolvectl", "default-route", "ppp0", "yes"]):
+      r = subprocess.run(cmd, capture_output=True, text=True)
+      if r.returncode != 0:
+        logging.warning(f"resolvectl failed ({' '.join(cmd[1:])}): {r.stderr.strip()}")
+        self.kill()
+        return False
+    logging.info(f"resolvectl: ppp0 DNS = {dns_servers}")
+    return True
+
+  @staticmethod
+  def cleanup_routes():
+    subprocess.run(["sudo", "ip", "route", "del", "default", "dev", "ppp0"], capture_output=True)
+    subprocess.run(["sudo", "ip", "route", "flush", "table", "1000"], capture_output=True)
+    # rules don't have a flush; delete until none remain
+    while subprocess.run(["sudo", "ip", "rule", "del", "table", "1000"], capture_output=True).returncode == 0:
+      pass
+    subprocess.run(["sudo", "resolvectl", "revert", "ppp0"], capture_output=True)
+
+
+class Modem:
+  def __init__(self):
+    self._ppp = PPPSession()
+    self._sim_change = False
+    self._apn = ""  # blank = network-provided via PCO
+    self._roaming_allowed = True
+    self.running = True
+    self.S = INITIAL_STATE.copy()
+
+  @staticmethod
+  def _read_param(key):
+    try:
+      with open(f"/data/params/d/{key}") as f:
+        return f.read().strip()
+    except FileNotFoundError:
+      return ""
+
+  @staticmethod
+  def _parse_reg(v: str) -> str:
+    try:
+      return CREG.get(int(v.split(",")[1].strip('"')), "unknown")
+    except (ValueError, IndexError):
+      return "unknown"
+
+  @staticmethod
+  def _has_modem_manager() -> bool:
+    return os.path.isfile("/lib/systemd/system/ModemManager.service")
+
+  def _is_roaming_allowed(self) -> bool:
+    if self.S["iccid"].startswith(WEBBING_ICCID_PREFIX):
+      return True
+    return self._read_param("GsmRoaming") == "1"
+
+  def _publish_state(self, **kwargs):
+    self.S.update(kwargs)
+    self.S["seconds_since_boot"] = time.monotonic()
+    with tempfile.NamedTemporaryFile(mode="w", dir="/dev/shm", delete=False) as f:
+      json.dump(self.S, f, indent=2)
+    os.chmod(f.name, 0o644)
+    os.replace(f.name, STATE_PATH)
+
+  def _at(self, cmd):
+    """Send AT command, return response lines. [] on error or if LPA holds port."""
+    fd = os.open(AT_LOCK, os.O_CREAT | os.O_RDWR, 0o666)
+    try:
+      fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+      os.close(fd)
+      return []
+    try:
+      with Serial(AT_PORT, baudrate=9600, timeout=5) as ser:
+        ser.reset_input_buffer()
+        ser.write((cmd + "\r").encode())
+        lines = []
+        while True:
+          raw = ser.readline()
+          if not raw:
+            raise TimeoutError("AT timeout")
+          line = raw.decode(errors="ignore").strip()
+          if not line:
+            continue
+          if line == "OK":
+            break
+          if line == "ERROR" or line.startswith("+CME ERROR"):
+            raise RuntimeError(line)
+          lines.append(line)
+        return lines
+    except (RuntimeError, TimeoutError, OSError) as e:
+      logging.info(f"AT {cmd} failed: {e}")
+      return []
+    finally:
+      fcntl.flock(fd, fcntl.LOCK_UN)
+      os.close(fd)
+
+  def _atv(self, cmd, pfx):
+    for line in self._at(cmd):
+      if pfx in line and ":" in line:
+        return line.split(":", 1)[1].strip()
+    return None
+
+  def _init_at_channel(self) -> bool:
+    """Run AT_INIT and confirm ATE0 took effect. Returns False if echo is still on."""
+    for c in AT_INIT:
+      self._at(c)
+    r = self._at("AT+CGMI")
+    return bool(r) and not r[0].startswith("AT")
+
+  def _configure_modem(self, modem_version: str):
+    if not modem_version.startswith("EG25"):
+      return
+    cmds = [
+      # clear initial EPS bearer APN (some carriers reject the default)
+      'AT+CGDCONT=0,"IP",""',
+
+      # SIM hot swap
+      'AT+QSIMDET=1,0',
+      'AT+QSIMSTAT=1',
+
+      # configure modem as data-centric
+      'AT+QNVW=5280,0,"0102000000000000"',
+      'AT+QNVFW="/nv/item_files/ims/IMS_enable",00',
+      'AT+QNVFW="/nv/item_files/modem/mmode/ue_usage_setting",01',
+    ]
+    for c in cmds:
+      self._at(c)
+
+  def _do_initializing(self):
+    if not os.path.exists(AT_PORT):
+      return State.INITIALIZING
+    logging.info("port found, initializing")
+    self._ppp.kill()
+    self._ppp.cleanup_routes()
+
+    if not self._init_at_channel():
+      logging.warning("AT echo still on, retrying")
+      return State.INITIALIZING
+
+    identity = self._read_identity()
+    if not identity["iccid"] or not identity["imei"]:
+      logging.warning(f"identity read incomplete: {identity}, retrying")
+      return State.INITIALIZING
+
+    self._configure_modem(identity["modem_version"])
+
+    self.S.update(identity)
+    self._apn = self._read_param("GsmApn")
+    self._roaming_allowed = self._is_roaming_allowed()
+    # blank APN lets the carrier supply one via PCO
+    self._at(f'AT+CGDCONT={DIAL_CID},"IP","{self._apn}"')
+    logging.info(f"APN '{self._apn or '(network-provided)'}' written to CID {DIAL_CID}, roaming={'on' if self._roaming_allowed else 'off'}")
+
+    self._sim_change = False  # clear since we just re-read identity with the new SIM
+    self._publish_state(**identity)
+    return State.SEARCHING
+
+  def _read_identity(self):
+    def first_line(cmd):
+      r = self._at(cmd)
+      return r[0].strip() if r else ""
+
+    imei = first_line("AT+CGSN")
+    if not (imei.isdigit() and 14 <= len(imei) <= 17):  # 3GPP TS 23.003
+      imei = ""
+
+    iccid = (self._atv("AT+QCCID", "+QCCID:") or "").rstrip("F")
+    if not iccid.isdigit():
+      iccid = ""
+
+    imsi = first_line("AT+CIMI")
+    mcc_mnc = imsi[:6] if imsi.isdigit() and len(imsi) >= 6 else ""
+
+    modem_version = first_line("AT+GMR")
+
+    logging.info(f"imei={imei} iccid={iccid} mcc_mnc={mcc_mnc} ver={modem_version}")
+    return {"imei": imei, "iccid": iccid, "mcc_mnc": mcc_mnc, "modem_version": modem_version}
+
+  def _do_searching(self):
+    new_roaming = self._is_roaming_allowed()
+    if new_roaming != self._roaming_allowed:
+      logging.info(f"roaming changed: {self._roaming_allowed} -> {new_roaming}")
+      self._roaming_allowed = new_roaming
+
+    v = self._atv("AT+CREG?", "+CREG:")
+    if not v:
+      return self._searching_idle()
+
+    reg = self._parse_reg(v)
+    greg = self._parse_reg(self._atv("AT+CGREG?", "+CGREG:") or "")
+    logging.debug(f"creg={reg} cgreg={greg} roaming_allowed={self._roaming_allowed}")
+
+    if reg == "roaming" and not self._roaming_allowed:
+      self._publish_state(registration=reg)
+      return State.SEARCHING
+
+    if reg in ("home", "roaming") and greg in ("home", "roaming"):
+      self._publish_state(registration=reg)
+      return State.CONNECTING
+
+    if reg != self.S.get("registration"):
+      self._publish_state(registration=reg)
+    return self._searching_idle()
+
+  def _searching_idle(self):
+    if self._sim_change or not os.path.exists(AT_PORT):
+      logging.info(f"-> reconnecting (sim_change={self._sim_change} port={os.path.exists(AT_PORT)})")
+      return State.DISCONNECTING
+    return State.SEARCHING
+
+  def _do_connecting(self):
+    logging.info("starting pppd")
+    self._ppp.reset_fail_counter()
+    self._sim_change = False
+    self._ppp.start()
+    return State.CONNECTED
+
+  def _handle_pppd_exit(self):
+    if self._sim_change or not os.path.exists(AT_PORT):
+      return State.DISCONNECTING
+    give_up = self._ppp.record_fail()
+    if give_up:
+      logging.warning(f"PPP fail {self._ppp.fails}/{self._ppp.MAX_FAILS}, reconnecting")
+      return State.DISCONNECTING
+    logging.warning(f"PPP fail {self._ppp.fails}/{self._ppp.MAX_FAILS}, retrying")
+    self._ppp.reset_data_port()
+    if not os.path.exists(AT_PORT):
+      return State.DISCONNECTING
+    self._ppp.start()
+    return State.CONNECTED
+
+  def _params_changed(self) -> bool:
+    new_apn = self._read_param("GsmApn")
+    if new_apn != self._apn:
+      logging.info(f"GsmApn changed: '{self._apn}' -> '{new_apn}'")
+      return True
+    new_roaming = self._is_roaming_allowed()
+    if new_roaming != self._roaming_allowed:
+      logging.info(f"roaming changed: {self._roaming_allowed} -> {new_roaming}")
+      return True
+    return False
+
+  def _check_iccid(self, state):
+    if state in (State.INITIALIZING, State.DISCONNECTING) or not self.S["iccid"]:
+      return
+    iccid = (self._atv("AT+QCCID", "+QCCID:") or "").rstrip("F")
+    if iccid and iccid != self.S["iccid"]:
+      logging.warning(f"iccid changed: {self.S['iccid']} -> {iccid}")
+      self._sim_change = True
+
+  def _do_connected(self):
+    if self._ppp.has_exited():
+      return self._handle_pppd_exit()
+
+    if self._sim_change or not os.path.exists(AT_PORT) or self._params_changed():
+      return State.DISCONNECTING
+
+    self._poll()
+    return State.CONNECTED
+
+  def _do_disconnecting(self):
+    logging.warning("reconnecting")
+    self._publish_state(**INITIAL_STATE)
+    self._ppp.kill()
+    self._ppp.cleanup_routes()
+    self._ppp.reset_data_port()
+    self._sim_change = False
+    return State.INITIALIZING
+
+  def _poll_signal(self) -> dict:
+    v = self._atv("AT+CSQ", "+CSQ:")
+    if not v:
+      return {}
+    try:
+      rssi = int(v.split(",")[0])
+      if rssi == 99:
+        return {}
+      return {"signal_strength": rssi, "signal_quality": min(100, int(rssi / 31 * 100))}
+    except (ValueError, IndexError):
+      return {}
+
+  def _poll_operator(self) -> dict:
+    v = self._atv("AT+COPS?", "+COPS:")
+    if not v:
+      return {}
+    p = v.split(",")
+    out: dict = {}
+    try:
+      if len(p) >= 3:
+        out["operator"] = p[2].strip('"')
+      if len(p) >= 4:
+        out["network_type"] = NETWORK_TYPE.get(int(p[3]), "unknown")
+    except (ValueError, IndexError):
+      pass
+    return out
+
+  def _poll_band(self) -> dict:
+    v = self._atv("AT+QNWINFO", "+QNWINFO:")
+    if not v:
+      return {}
+    info = v.replace('"', '').split(",")
+    try:
+      if len(info) >= 4:
+        return {"band": info[2], "channel": int(info[3])}
+    except ValueError:
+      pass
+    return {}
+
+  def _poll_extra(self) -> dict:
+    v = self._atv('AT+QENG="servingcell"', "+QENG:")
+    return {"extra": v.replace('"', '')} if v else {}
+
+  def _poll_temps(self) -> dict:
+    v = self._atv("AT+QTEMP", "+QTEMP:")
+    if not v:
+      return {}
+    try:
+      return {"temperatures": [t for t in (int(x) for x in v.split(",") if x.strip()) if t != 255]}
+    except (ValueError, IndexError):
+      return {}
+
+  def _poll_iface(self) -> dict:
+    try:
+      r = subprocess.run(["ip", "-4", "addr", "show", "ppp0"], capture_output=True, text=True, timeout=2)
+      ip, peer = "", ""
+      for line in r.stdout.splitlines():
+        # `inet 10.x.x.x peer 10.64.64.64/32 ...`
+        parts = line.strip().split()
+        if "inet" in parts:
+          i = parts.index("inet")
+          ip = parts[i + 1].split("/")[0]
+          if "peer" in parts:
+            peer = parts[parts.index("peer") + 1].split("/")[0]
+          break
+      if ip:
+        if self._ppp.maybe_install_routes(ip, peer):
+          self._ppp.maybe_install_dns(self._read_cellular_dns())
+        return {"ip_address": ip, "connected": True}
+      if self.S["connected"]:
+        return {"connected": False, "ip_address": ""}
+    except Exception:
+      pass
+    return {}
+
+  def _read_cellular_dns(self) -> list[str]:
+    v = self._atv(f"AT+CGCONTRDP={DIAL_CID}", "+CGCONTRDP:")
+    if not v:
+      return []
+    # +CGCONTRDP: <cid>,<bearer_id>,<apn>,<local_addr>,<gw_addr>,<dns_prim>,<dns_sec>,...
+    fields = [f.strip().strip('"') for f in v.split(",")]
+    dns_servers = []
+    for d in fields[5:7]:
+      try:
+        dns_servers.append(str(IPv4Address(d)))
+      except (AddressValueError, ValueError):
+        pass
+    if not dns_servers:
+      dns_servers = [
+        "8.8.8.8",  # Google
+        "1.1.1.1",  # Cloudflare
+      ]
+      logging.warning(f"no cellular DNS servers reported by modem: {v!r}; using fallback {dns_servers}")
+    return dns_servers
+
+  def _poll_byte_counters(self) -> dict:
+    try:
+      with open("/sys/class/net/ppp0/statistics/tx_bytes") as f:
+        tx = int(f.read().strip())
+      with open("/sys/class/net/ppp0/statistics/rx_bytes") as f:
+        rx = int(f.read().strip())
+    except Exception:
+      return {}
+    return {"tx_bytes": tx, "rx_bytes": rx}
+
+  def _poll(self):
+    s: dict = {}
+    for fn in (self._poll_signal, self._poll_operator, self._poll_band,
+               self._poll_extra, self._poll_temps, self._poll_iface,
+               self._poll_byte_counters):
+      s.update(fn())
+    if s:
+      self._publish_state(**s)
+
+  def run(self):
+    logging.info("starting")
+    self._publish_state(state=State.INITIALIZING.value)
+    if self._has_modem_manager():
+      subprocess.run(["sudo", "systemctl", "mask", "--runtime", "ModemManager"], capture_output=True)
+      subprocess.run(["sudo", "systemctl", "stop", "ModemManager"], capture_output=True)
+    self._ppp.kill()
+
+    state = State.INITIALIZING
+
+    handlers = {
+      State.INITIALIZING: self._do_initializing,
+      State.SEARCHING: self._do_searching,
+      State.CONNECTING: self._do_connecting,
+      State.CONNECTED: self._do_connected,
+      State.DISCONNECTING: self._do_disconnecting,
+    }
+
+    while self.running:
+      try:
+        self._check_iccid(state)
+        prev = state
+        state = handlers[state]()
+        if state != prev:
+          self._publish_state(state=state.value)
+          logging.info(f"{prev.value} -> {state.value}")
+      except Exception:
+        logging.exception(f"error in {state.value}")
+        state = State.DISCONNECTING
+      time.sleep(STATE_WAIT)
+
+  def stop(self):
+    self.running = False
+    self._ppp.kill()
+    self._ppp.cleanup_routes()
+    try:
+      os.remove(STATE_PATH)
+    except FileNotFoundError:
+      pass
+    if self._has_modem_manager():
+      subprocess.run(["sudo", "systemctl", "unmask", "--runtime", "ModemManager"], capture_output=True)
+      subprocess.run(["sudo", "systemctl", "start", "ModemManager"], capture_output=True)
+
+
+def main():
+  m = Modem()
+
+  def _sig(*_):
+    m.running = False
+
+  signal.signal(signal.SIGINT, _sig)
+  signal.signal(signal.SIGTERM, _sig)
+  m.run()
+  m.stop()
+
+
+if __name__ == "__main__":
+  main()

openpilot/common/hardware/tici/pins.py

@@ -0,0 +1,27 @@
+# GPIO pin definitions
+class GPIO:
+  # both GPIO_STM_RST_N and GPIO_LTE_RST_N are misnamed, they are high to reset
+  HUB_RST_N = 30
+  UBLOX_RST_N = 32
+  UBLOX_SAFEBOOT_N = 33
+  GNSS_PWR_EN = 34 # SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN
+
+  STM_RST_N = 124
+  STM_BOOT0 = 134
+  STM_PWR_EN_N = 41 # because STM32H7 RST doesn't generate a full power-on-reset
+
+  SIREN = 42
+  SOM_ST_IO = 49
+
+  LTE_RST_N = 50
+  LTE_PWRKEY = 116
+  LTE_BOOT = 52
+
+  # GPIO_CAM0_DVDD_EN = /sys/kernel/debug/regulator/camera_rear_ldo
+  CAM0_AVDD_EN = 8
+  CAM0_RSTN = 9
+  CAM1_RSTN = 7
+  CAM2_RSTN = 12
+
+  # Sensor interrupts
+  LSM_INT = 84

openpilot/common/hardware/tici/power_monitor.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+import sys
+import time
+import datetime
+import numpy as np
+from collections import deque
+
+from openpilot.common.realtime import Ratekeeper
+from openpilot.common.filter_simple import FirstOrderFilter
+
+
+def read_power():
+  with open("/sys/bus/i2c/devices/0-0040/hwmon/hwmon1/power1_input") as f:
+    return int(f.read()) / 1e6
+
+def sample_power(seconds=5) -> list[float]:
+  rate = 123
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+
+  pwrs = []
+  for _ in range(rate*seconds):
+    pwrs.append(read_power())
+    rk.keep_time()
+  return pwrs
+
+def get_power(seconds=5):
+  pwrs = sample_power(seconds)
+  return np.mean(pwrs)
+
+def wait_for_power(min_pwr, max_pwr, min_secs_in_range, timeout):
+  start_time = time.monotonic()
+  pwrs = deque([min_pwr - 1.]*min_secs_in_range, maxlen=min_secs_in_range)
+  while (time.monotonic() - start_time < timeout):
+    pwrs.append(get_power(1))
+    if all(min_pwr <= p <= max_pwr for p in pwrs):
+      break
+  return np.mean(pwrs)
+
+
+if __name__ == "__main__":
+  duration = None
+  if len(sys.argv) > 1:
+    duration = int(sys.argv[1])
+
+  rate = 23
+  rk = Ratekeeper(rate, print_delay_threshold=None)
+  fltr = FirstOrderFilter(0, 5, 1. / rate, initialized=False)
+
+  measurements = []
+  start_time = time.monotonic()
+
+  try:
+    while duration is None or time.monotonic() - start_time < duration:
+      fltr.update(read_power())
+      if rk.frame % rate == 0:
+        measurements.append(fltr.x)
+        t = datetime.timedelta(seconds=time.monotonic() - start_time)
+        avg = sum(measurements) / len(measurements)
+        print(f"Now: {fltr.x:.2f} W, Avg: {avg:.2f} W over {t}")
+      rk.keep_time()
+  except KeyboardInterrupt:
+    pass
+
+  t = datetime.timedelta(seconds=time.monotonic() - start_time)
+  avg = sum(measurements) / len(measurements)
+  print(f"\nAverage power: {avg:.2f}W over {t}")

openpilot/common/hardware/tici/tests/__init__.py

@@ -0,0 +1 @@
+

openpilot/common/hardware/tici/tests/test_agnos_updater.py

@@ -0,0 +1,20 @@
+import json
+import os
+import requests
+
+TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)))
+MANIFEST = os.path.join(TEST_DIR, "../agnos.json")
+
+
+class TestAgnosUpdater:
+
+  def test_manifest(self):
+    with open(MANIFEST) as f:
+      m = json.load(f)
+
+    for img in m:
+      r = requests.head(img['url'], timeout=10)
+      r.raise_for_status()
+      assert r.headers['Content-Type'] == "application/x-xz"
+      if not img['sparse']:
+        assert img['hash'] == img['hash_raw']

openpilot/common/hardware/tici/tests/test_amplifier.py

@@ -0,0 +1,68 @@
+import pytest
+import time
+import subprocess
+
+from panda import Panda
+from openpilot.common.hardware import TICI, HARDWARE
+from openpilot.common.hardware.tici.amplifier import Amplifier
+
+
+class TestAmplifier:
+
+  @classmethod
+  def setup_class(cls):
+    if not TICI:
+      pytest.skip()
+
+  def setup_method(self):
+    # clear dmesg
+    subprocess.check_call("sudo dmesg -C", shell=True)
+
+    HARDWARE.reset_internal_panda()
+    Panda.wait_for_panda(None, 30)
+    self.panda = Panda()
+
+  def teardown_method(self):
+    HARDWARE.reset_internal_panda()
+
+  def _check_for_i2c_errors(self, expected):
+    dmesg = subprocess.check_output("dmesg", shell=True, encoding='utf8')
+    i2c_lines = [l for l in dmesg.strip().splitlines() if 'i2c_geni a88000.i2c' in l]
+    i2c_str = '\n'.join(i2c_lines)
+
+    if not expected:
+      return len(i2c_lines) == 0
+    else:
+      return "i2c error :-107" in i2c_str or "Bus arbitration lost" in i2c_str
+
+  def test_init(self):
+    amp = Amplifier(debug=True)
+    r = amp.initialize_configuration()
+    assert r
+    assert self._check_for_i2c_errors(False)
+
+  def test_shutdown(self):
+    amp = Amplifier(debug=True)
+    for _ in range(10):
+      r = amp.set_global_shutdown(True)
+      r = amp.set_global_shutdown(False)
+      # amp config should be successful, with no i2c errors
+      assert r
+      assert self._check_for_i2c_errors(False)
+
+  def test_init_while_siren_play(self):
+    for _ in range(10):
+      self.panda.set_siren(False)
+      time.sleep(0.1)
+
+      self.panda.set_siren(True)
+      time.sleep(0.1)
+
+      amp = Amplifier(debug=True)
+      r = amp.initialize_configuration()
+      assert r
+
+      if self._check_for_i2c_errors(True):
+        break
+    else:
+      pytest.fail("didn't hit any i2c errors")

openpilot/common/hardware/usb.py

@@ -0,0 +1,73 @@
+from pathlib import Path
+
+CHESTNUT_VENDOR_ID = 0xADD1
+CHESTNUT_PRODUCT_ID = 0x0001
+USB_DEVICES_PATH = Path("/sys/bus/usb/devices")
+
+
+def read(path: Path) -> str | None:
+  try:
+    return path.read_text().strip()
+  except OSError:
+    return None
+
+
+def read_int(path: Path, base: int = 10) -> int:
+  try:
+    return int(path.read_text(), base)
+  except (OSError, ValueError):
+    return 0
+
+
+def usb_devices() -> list[Path]:
+  try:
+    devices = (d for d in USB_DEVICES_PATH.glob("*") if (d / "idVendor").exists())
+    return sorted(devices, key=lambda p: p.name)
+  except OSError:
+    return []
+
+
+def controller(device: Path) -> Path | None:
+  try:
+    return next((parent for parent in device.resolve().parents if parent.name.endswith(".ssusb")), None)
+  except OSError:
+    return None
+
+
+def get_usb_state() -> list[dict]:
+  devices = []
+  for device in usb_devices():
+    vendor_id = read_int(device / "idVendor", 16)
+    product_id = read_int(device / "idProduct", 16)
+    ctrl = controller(device)
+    devices.append({
+      "busnum": read_int(device / "busnum"),
+      "devnum": read_int(device / "devnum"),
+      "vendorId": vendor_id,
+      "productId": product_id,
+      "speedMbps": read_int(device / "speed"),
+      "manufacturer": read(device / "manufacturer") or "",
+      "product": read(device / "product") or "",
+      "linkErrorCount": read_int(ctrl / "portli", 0) & 0xFFFF if ctrl is not None else 0,
+    })
+  return devices
+
+
+def set_usb_state(device_state, devices: list[dict]) -> None:
+  entries = device_state.usbState.init('devices', len(devices))
+
+  chestnut_present = False
+  for entry, device in zip(entries, devices, strict=True):
+    entry.busnum = device["busnum"]
+    entry.devnum = device["devnum"]
+    entry.vendorId = device["vendorId"]
+    entry.productId = device["productId"]
+    entry.speedMbps = device["speedMbps"]
+    entry.manufacturer = device["manufacturer"]
+    entry.product = device["product"]
+    entry.linkErrorCount = device["linkErrorCount"]
+
+    if (entry.vendorId, entry.productId) == (CHESTNUT_VENDOR_ID, CHESTNUT_PRODUCT_ID):
+      chestnut_present = True
+
+  device_state.chestnutPresent = chestnut_present

openpilot/common/i2c.py

@@ -0,0 +1,81 @@
+import os
+import fcntl
+import ctypes
+
+# I2C constants from /usr/include/linux/i2c-dev.h
+I2C_SLAVE = 0x0703
+I2C_SLAVE_FORCE = 0x0706
+I2C_SMBUS = 0x0720
+
+# SMBus transfer types
+I2C_SMBUS_READ = 1
+I2C_SMBUS_WRITE = 0
+I2C_SMBUS_BYTE_DATA = 2
+I2C_SMBUS_I2C_BLOCK_DATA = 8
+
+I2C_SMBUS_BLOCK_MAX = 32
+
+
+class _I2cSmbusData(ctypes.Union):
+  _fields_ = [
+    ("byte", ctypes.c_uint8),
+    ("word", ctypes.c_uint16),
+    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
+  ]
+
+
+class _I2cSmbusIoctlData(ctypes.Structure):
+  _fields_ = [
+    ("read_write", ctypes.c_uint8),
+    ("command", ctypes.c_uint8),
+    ("size", ctypes.c_uint32),
+    ("data", ctypes.POINTER(_I2cSmbusData)),
+  ]
+
+
+class SMBus:
+  def __init__(self, bus: int):
+    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
+
+  def __enter__(self) -> 'SMBus':
+    return self
+
+  def __exit__(self, *args) -> None:
+    self.close()
+
+  def close(self) -> None:
+    if hasattr(self, '_fd') and self._fd >= 0:
+      os.close(self._fd)
+      self._fd = -1
+
+  def _set_address(self, addr: int, force: bool = False) -> None:
+    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
+    fcntl.ioctl(self._fd, ioctl_arg, addr)
+
+  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
+    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
+    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
+
+  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
+    return int(data.byte)
+
+  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    data.byte = value & 0xFF
+    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
+
+  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
+    self._set_address(addr, force)
+    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
+      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
+
+    data = _I2cSmbusData()
+    data.block[0] = length
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
+    read_len = int(data.block[0]) or length
+    read_len = min(read_len, length)
+    return [int(b) for b in data.block[1 : read_len + 1]]

openpilot/common/linux.py

@@ -0,0 +1,50 @@
+class LinuxSystemStats:
+  def __init__(self) -> None:
+    self._last_cpu_times = self._read_cpu_times()
+
+  @staticmethod
+  def _read_cpu_times() -> dict[int, tuple[int, int]]:
+    cpu_times = {}
+    with open('/proc/stat') as f:
+      for line in f:
+        name, *values = line.split()
+        if not name.startswith('cpu') or not name[3:].isdigit():
+          continue
+
+        times = [int(value) for value in values]
+        idle = sum(times[3:5])
+        total = sum(times[:8])
+        cpu_times[int(name[3:])] = (idle, total)
+    return cpu_times
+
+  def cpu_usage_percent(self) -> list[float]:
+    current_cpu_times = self._read_cpu_times()
+    usage = []
+    for cpu, (idle, total) in sorted(current_cpu_times.items()):
+      last_times = self._last_cpu_times.get(cpu)
+      if last_times is None:
+        usage.append(0.)
+        continue
+
+      last_idle, last_total = last_times
+      idle_delta = idle - last_idle
+      total_delta = total - last_total
+      if idle_delta < 0 or total_delta <= 0:
+        usage.append(0.)
+      else:
+        usage.append(max(0., min(100., 100. * (total_delta - idle_delta) / total_delta)))
+
+    self._last_cpu_times = current_cpu_times
+    return usage
+
+  @staticmethod
+  def memory_usage_percent() -> float:
+    memory = {}
+    with open('/proc/meminfo') as f:
+      for line in f:
+        key, value, *_ = line.split()
+        if key in ('MemTotal:', 'MemAvailable:'):
+          memory[key] = int(value)
+
+    total = memory['MemTotal:']
+    return max(0., min(100., 100. * (total - memory['MemAvailable:']) / total))

openpilot/common/logging_extra.py

@@ -0,0 +1,249 @@
+import io
+import os
+import sys
+import copy
+import json
+import time
+import uuid
+import socket
+import logging
+import traceback
+import numpy as np
+from threading import local
+from collections import OrderedDict
+from contextlib import contextmanager
+
+LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
+
+def json_handler(obj):
+  if isinstance(obj, np.bool_):
+    return bool(obj)
+  # if isinstance(obj, (datetime.date, datetime.time)):
+  #   return obj.isoformat()
+  return repr(obj)
+
+def json_robust_dumps(obj):
+  return json.dumps(obj, default=json_handler)
+
+class NiceOrderedDict(OrderedDict):
+  def __str__(self):
+    return json_robust_dumps(self)
+
+class SwagFormatter(logging.Formatter):
+  def __init__(self, swaglogger):
+    logging.Formatter.__init__(self, None, '%a %b %d %H:%M:%S %Z %Y')
+
+    self.swaglogger = swaglogger
+    self.host = socket.gethostname()
+
+  def format_dict(self, record):
+    record_dict = NiceOrderedDict()
+
+    if isinstance(record.msg, dict):
+      record_dict['msg'] = record.msg
+    else:
+      try:
+        record_dict['msg'] = record.getMessage()
+      except (ValueError, TypeError):
+        record_dict['msg'] = [record.msg]+record.args
+
+    record_dict['ctx'] = self.swaglogger.get_ctx()
+
+    if record.exc_info:
+      record_dict['exc_info'] = self.formatException(record.exc_info)
+
+    record_dict['level'] = record.levelname
+    record_dict['levelnum'] = record.levelno
+    record_dict['name'] = record.name
+    record_dict['filename'] = record.filename
+    record_dict['lineno'] = record.lineno
+    record_dict['pathname'] = record.pathname
+    record_dict['module'] = record.module
+    record_dict['funcName'] = record.funcName
+    record_dict['host'] = self.host
+    record_dict['process'] = record.process
+    record_dict['thread'] = record.thread
+    record_dict['threadName'] = record.threadName
+    record_dict['created'] = record.created
+
+    return record_dict
+
+  def format(self, record):
+    if self.swaglogger is None:
+      raise Exception("must set swaglogger before calling format()")
+    return json_robust_dumps(self.format_dict(record))
+
+class SwagLogFileFormatter(SwagFormatter):
+  def fix_kv(self, k, v):
+    # append type to names to preserve legacy naming in logs
+    # avoids overlapping key namespaces with different types
+    # e.g. log.info() creates 'msg' -> 'msg$s'
+    #      log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
+    #      because overlapping namespace 'msg' caused problems
+    if isinstance(v, (str, bytes)):
+      k += "$s"
+    elif isinstance(v, float):
+      k += "$f"
+    elif isinstance(v, bool):
+      k += "$b"
+    elif isinstance(v, int):
+      k += "$i"
+    elif isinstance(v, dict):
+      nv = {}
+      for ik, iv in v.items():
+        ik, iv = self.fix_kv(ik, iv)
+        nv[ik] = iv
+      v = nv
+    elif isinstance(v, list):
+      k += "$a"
+    return k, v
+
+  def format(self, record):
+    if isinstance(record, str):
+      v = json.loads(record)
+    else:
+      v = self.format_dict(record)
+
+    mk, mv = self.fix_kv('msg', v['msg'])
+    del v['msg']
+    v[mk] = mv
+    v['id'] = uuid.uuid4().hex
+
+    return json_robust_dumps(v)
+
+class SwagErrorFilter(logging.Filter):
+  def filter(self, record):
+    return record.levelno < logging.ERROR
+
+def _tmpfunc():
+  return 0
+
+def _srcfile():
+  return os.path.normcase(_tmpfunc.__code__.co_filename)
+
+class SwagLogger(logging.Logger):
+  def __init__(self):
+    logging.Logger.__init__(self, "swaglog")
+
+    self.global_ctx = {}
+
+    self.log_local = local()
+    self.log_local.ctx = {}
+
+  def local_ctx(self):
+    try:
+      return self.log_local.ctx
+    except AttributeError:
+      self.log_local.ctx = {}
+      return self.log_local.ctx
+
+  def get_ctx(self):
+    return dict(self.local_ctx(), **self.global_ctx)
+
+  @contextmanager
+  def ctx(self, **kwargs):
+    old_ctx = self.local_ctx()
+    self.log_local.ctx = copy.copy(old_ctx) or {}
+    self.log_local.ctx.update(kwargs)
+    try:
+      yield
+    finally:
+      self.log_local.ctx = old_ctx
+
+  def bind(self, **kwargs):
+    self.local_ctx().update(kwargs)
+
+  def bind_global(self, **kwargs):
+    self.global_ctx.update(kwargs)
+
+  def event(self, event, *args, **kwargs):
+    evt = NiceOrderedDict()
+    evt['event'] = event
+    if args:
+      evt['args'] = args
+    evt.update(kwargs)
+    if 'error' in kwargs:
+      self.error(evt)
+    elif 'debug' in kwargs:
+      self.debug(evt)
+    else:
+      self.info(evt)
+
+  def timestamp(self, event_name):
+    if LOG_TIMESTAMPS:
+      t = time.monotonic()
+      tstp = NiceOrderedDict()
+      tstp['timestamp'] = NiceOrderedDict()
+      tstp['timestamp']["event"] = event_name
+      tstp['timestamp']["time"] = t*1e9
+      self.debug(tstp)
+
+  def findCaller(self, stack_info=False, stacklevel=1):
+    """
+    Find the stack frame of the caller so that we can note the source
+    file name, line number and function name.
+    """
+    f = sys._getframe(3)
+    #On some versions of IronPython, currentframe() returns None if
+    #IronPython isn't run with -X:Frames.
+    if f is not None:
+      f = f.f_back
+    orig_f = f
+    while f and stacklevel > 1:
+      f = f.f_back
+      stacklevel -= 1
+    if not f:
+      f = orig_f
+    rv = "(unknown file)", 0, "(unknown function)", None
+    while hasattr(f, "f_code"):
+      co = f.f_code
+      filename = os.path.normcase(co.co_filename)
+
+      if filename == _srcfile:
+        f = f.f_back
+        continue
+      sinfo = None
+      if stack_info:
+        sio = io.StringIO()
+        sio.write('Stack (most recent call last):\n')
+        traceback.print_stack(f, file=sio)
+        sinfo = sio.getvalue()
+        if sinfo[-1] == '\n':
+          sinfo = sinfo[:-1]
+        sio.close()
+      rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
+      break
+    return rv
+
+if __name__ == "__main__":
+  log = SwagLogger()
+
+  stdout_handler = logging.StreamHandler(sys.stdout)
+  stdout_handler.setLevel(logging.INFO)
+  stdout_handler.addFilter(SwagErrorFilter())
+  log.addHandler(stdout_handler)
+
+  stderr_handler = logging.StreamHandler(sys.stderr)
+  stderr_handler.setLevel(logging.ERROR)
+  log.addHandler(stderr_handler)
+
+  log.info("asdasd %s", "a")
+  log.info({'wut': 1})
+  log.warning("warning")
+  log.error("error")
+  log.critical("critical")
+  log.event("test", x="y")
+
+  with log.ctx():
+    stdout_handler.setFormatter(SwagFormatter(log))
+    stderr_handler.setFormatter(SwagFormatter(log))
+    log.bind(user="some user")
+    log.info("in req")
+    print("")
+    log.warning("warning")
+    print("")
+    log.error("error")
+    print("")
+    log.critical("critical")
+    print("")
+    log.event("do_req", a=1, b="c")

openpilot/common/markdown.py

@@ -0,0 +1,45 @@
+HTML_REPLACEMENTS = [
+  (r'&', r'&'),
+  (r'"', r'"'),
+]
+
+def parse_markdown(text: str, tab_length: int = 2) -> str:
+  lines = text.split("\n")
+  output: list[str] = []
+  list_level = 0
+
+  def end_outstanding_lists(level: int, end_level: int) -> int:
+    while level > end_level:
+      level -= 1
+      output.append("</ul>")
+      if level > 0:
+        output.append("</li>")
+    return end_level
+
+  for i, line in enumerate(lines):
+    if i + 1 < len(lines) and lines[i + 1].startswith("==="):  # heading
+      output.append(f"<h1>{line}</h1>")
+    elif line.startswith("==="):
+      pass
+    elif line.lstrip().startswith("* "):  # list
+      line_level = 1 + line.count(" " * tab_length, 0, line.index("*"))
+      if list_level >= line_level:
+        list_level = end_outstanding_lists(list_level, line_level)
+      else:
+        list_level += 1
+        if list_level > 1:
+          output[-1] = output[-1].replace("</li>", "")
+        output.append("<ul>")
+      output.append(f"<li>{line.replace('*', '', 1).lstrip()}</li>")
+    else:
+      list_level = end_outstanding_lists(list_level, 0)
+      if len(line) > 0:
+        output.append(line)
+
+  end_outstanding_lists(list_level, 0)
+  output_str = "\n".join(output) + "\n"
+
+  for (fr, to) in HTML_REPLACEMENTS:
+    output_str = output_str.replace(fr, to)
+
+  return output_str

openpilot/common/mock/__init__.py

@@ -0,0 +1,50 @@
+"""
+Utilities for generating mock messages for testing.
+example in openpilot/common/tests/test_mock.py
+"""
+
+
+import functools
+import threading
+from openpilot.cereal.messaging import PubMaster
+from openpilot.cereal.services import SERVICE_LIST
+from openpilot.common.mock.generators import generate_livePose
+from openpilot.common.realtime import Ratekeeper
+
+
+MOCK_GENERATOR = {
+  "livePose": generate_livePose
+}
+
+
+def generate_messages_loop(services: list[str], done: threading.Event):
+  pm = PubMaster(services)
+  rk = Ratekeeper(100)
+  i = 0
+  while not done.is_set():
+    for s in services:
+      should_send = i % (100/SERVICE_LIST[s].frequency) == 0
+      if should_send:
+        message = MOCK_GENERATOR[s]()
+        pm.send(s, message)
+    i += 1
+    rk.keep_time()
+
+
+def mock_messages(services: list[str] | str):
+  if isinstance(services, str):
+    services = [services]
+
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      done = threading.Event()
+      t = threading.Thread(target=generate_messages_loop, args=(services, done))
+      t.start()
+      try:
+        return func(*args, **kwargs)
+      finally:
+        done.set()
+        t.join()
+    return wrapper
+  return decorator

openpilot/common/mock/generators.py

@@ -0,0 +1,14 @@
+from openpilot.cereal import messaging
+
+
+def generate_livePose():
+  msg = messaging.new_message('livePose')
+  meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
+  msg.livePose.orientationNED = meas
+  msg.livePose.velocityDevice = meas
+  msg.livePose.angularVelocityDevice = meas
+  msg.livePose.accelerationDevice = meas
+  msg.livePose.inputsOK = True
+  msg.livePose.posenetOK = True
+  msg.livePose.sensorsOK = True
+  return msg

openpilot/common/parameterized.py

@@ -0,0 +1,54 @@
+import re
+import sys
+import pytest
+import inspect
+
+
+def _to_safe_name(s):
+  return re.sub(r"[^a-zA-Z0-9_]+", "_", str(s)).strip("_")
+
+
+class parameterized:
+  @staticmethod
+  def expand(cases):
+    cases = list(cases)
+
+    if not cases:
+      return lambda func: pytest.mark.skip("no parameterized cases")(func)
+
+    def decorator(func):
+      params = [p for p in inspect.signature(func).parameters if p != 'self']
+      normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
+      # Infer arg count from first case so extra params (e.g. from @given) are left untouched
+      expand_params = params[: len(normalized[0])]
+      if len(expand_params) == 1:
+        return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
+      return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
+
+    return decorator
+
+
+def parameterized_class(attrs, input_list=None):
+  if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
+    params_list = attrs
+  else:
+    assert input_list is not None
+    attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
+    params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
+
+  def decorator(cls):
+    globs = sys._getframe(1).f_globals
+    for i, params in enumerate(params_list):
+      # append sanitized string param values so pytest -k can filter by them
+      suffix = "_".join(filter(None, (_to_safe_name(v) for v in params.values() if isinstance(v, str))))
+      name = f"{cls.__name__}_{i}" + (f"_{suffix}" if suffix else "")
+      new_cls = type(name, (cls,), dict(params))
+      new_cls.__module__ = cls.__module__
+      new_cls.__test__ = True  # override inherited False so pytest collects this subclass
+      globs[name] = new_cls
+    # Don't collect the un-parametrised base, but return it so outer decorators
+    # (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
+    cls.__test__ = False
+    return cls
+
+  return decorator

openpilot/common/params.cc

@@ -0,0 +1,244 @@
+#include "common/params.h"
+
+#include <dirent.h>
+#include <sys/file.h>
+
+#include <algorithm>
+#include <cassert>
+#include <csignal>
+#include <unordered_map>
+
+#include "common/params_keys.h"
+#include "common/queue.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+#include "common/hardware/hw.h"
+
+namespace {
+
+volatile sig_atomic_t params_do_exit = 0;
+void params_sig_handler(int signal) {
+  params_do_exit = 1;
+}
+
+int fsync_dir(const std::string &path) {
+  int result = -1;
+  int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
+  if (fd >= 0) {
+    result = HANDLE_EINTR(fsync(fd));
+    HANDLE_EINTR(close(fd));
+  }
+  return result;
+}
+
+bool create_params_path(const std::string &param_path, const std::string &key_path) {
+  // Make sure params path exists
+  if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
+    return false;
+  }
+
+  // See if the symlink exists, otherwise create it
+  if (!util::file_exists(key_path)) {
+    // 1) Create temp folder
+    // 2) Symlink it to temp link
+    // 3) Move symlink to <params>/d
+
+    std::string tmp_path = param_path + "/.tmp_XXXXXX";
+    // this should be OK since mkdtemp just replaces characters in place
+    char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
+    if (tmp_dir == NULL) {
+      return false;
+    }
+
+    std::string link_path = std::string(tmp_dir) + ".link";
+    if (symlink(tmp_dir, link_path.c_str()) != 0) {
+      return false;
+    }
+
+    // don't return false if it has been created by other
+    if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
+  std::string params_path = path.empty() ? Path::params() : path;
+  if (!create_params_path(params_path, params_path + prefix)) {
+    throw std::runtime_error(util::string_format(
+        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
+        errno, params_path.c_str(), prefix.c_str()));
+  }
+  return params_path;
+}
+
+class FileLock {
+public:
+  FileLock(const std::string &fn) {
+    fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
+    if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
+      LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
+    }
+  }
+  ~FileLock() { close(fd_); }
+
+private:
+  int fd_ = -1;
+};
+
+} // namespace
+
+
+Params::Params(const std::string &path) {
+  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
+  params_path = ensure_params_path(params_prefix, path);
+}
+
+Params::~Params() {
+  if (future.valid()) {
+    future.wait();
+  }
+  assert(queue.empty());
+}
+
+std::vector<std::string> Params::allKeys(ParamKeyFlag flag) const {
+  std::vector<std::string> ret;
+  for (auto &p : keys) {
+    if (flag == ALL || (p.second.flags & flag)) {
+      ret.push_back(p.first);
+    }
+  }
+  return ret;
+}
+
+bool Params::checkKey(const std::string &key) {
+  return keys.find(key) != keys.end();
+}
+
+ParamKeyFlag Params::getKeyFlag(const std::string &key) {
+  return static_cast<ParamKeyFlag>(keys[key].flags);
+}
+
+ParamKeyType Params::getKeyType(const std::string &key) {
+  return keys[key].type;
+}
+
+std::optional<std::string> Params::getKeyDefaultValue(const std::string &key) {
+  return keys[key].default_value;
+}
+
+int Params::put(const char* key, const char* value, size_t value_size) {
+  // Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
+  // 1) Create temp file
+  // 2) Write data to temp file
+  // 3) fsync() the temp file
+  // 4) rename the temp file to the real name
+  // 5) fsync() the containing directory
+  std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
+  int tmp_fd = mkstemp((char*)tmp_path.c_str());
+  if (tmp_fd < 0) return -1;
+
+  int result = -1;
+  do {
+    // Write value to temp.
+    ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
+    if (bytes_written < 0 || (size_t)bytes_written != value_size) {
+      result = -20;
+      break;
+    }
+
+    // fsync to force persist the changes.
+    if ((result = HANDLE_EINTR(fsync(tmp_fd))) < 0) break;
+
+    FileLock file_lock(params_path + "/.lock");
+
+    // Move temp into place.
+    if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
+
+    // fsync parent directory
+    result = fsync_dir(getParamPath());
+  } while (false);
+
+  close(tmp_fd);
+  if (result != 0) {
+    ::unlink(tmp_path.c_str());
+  }
+  return result;
+}
+
+int Params::remove(const std::string &key) {
+  FileLock file_lock(params_path + "/.lock");
+  int result = unlink(getParamPath(key).c_str());
+  if (result != 0) {
+    return result;
+  }
+  return fsync_dir(getParamPath());
+}
+
+std::string Params::get(const std::string &key, bool block) {
+  if (!block) {
+    return util::read_file(getParamPath(key));
+  } else {
+    // blocking read until successful
+    params_do_exit = 0;
+    void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
+    void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
+
+    std::string value;
+    while (!params_do_exit) {
+      if (value = util::read_file(getParamPath(key)); !value.empty()) {
+        break;
+      }
+      util::sleep_for(100);  // 0.1 s
+    }
+
+    std::signal(SIGINT, prev_handler_sigint);
+    std::signal(SIGTERM, prev_handler_sigterm);
+    return value;
+  }
+}
+
+std::map<std::string, std::string> Params::readAll() {
+  FileLock file_lock(params_path + "/.lock");
+  return util::read_files_in_dir(getParamPath());
+}
+
+void Params::clearAll(ParamKeyFlag key_flag) {
+  FileLock file_lock(params_path + "/.lock");
+
+  // 1) delete params of key_flag
+  // 2) delete files that are not defined in the keys.
+  if (DIR *d = opendir(getParamPath().c_str())) {
+    struct dirent *de = NULL;
+    while ((de = readdir(d))) {
+      if (de->d_type != DT_DIR) {
+        auto it = keys.find(de->d_name);
+        if (it == keys.end() || (it->second.flags & key_flag)) {
+          unlink(getParamPath(de->d_name).c_str());
+        }
+      }
+    }
+    closedir(d);
+  }
+
+  fsync_dir(getParamPath());
+}
+
+void Params::putNonBlocking(const std::string &key, const std::string &val) {
+   queue.push(std::make_pair(key, val));
+  // start thread on demand
+  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
+    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
+  }
+}
+
+void Params::asyncWriteThread() {
+  // TODO: write the latest one if a key has multiple values in the queue.
+  std::pair<std::string, std::string> p;
+  while (queue.try_pop(p, 0)) {
+    // Params::put is Thread-Safe
+    put(p.first, p.second);
+  }
+}

openpilot/common/params.h

@@ -0,0 +1,91 @@
+#pragma once
+
+#include <future>
+#include <map>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "common/queue.h"
+
+enum ParamKeyFlag {
+  PERSISTENT = 0x02,
+  CLEAR_ON_MANAGER_START = 0x04,
+  CLEAR_ON_ONROAD_TRANSITION = 0x08,
+  CLEAR_ON_OFFROAD_TRANSITION = 0x10,
+  DONT_LOG = 0x20,
+  DEVELOPMENT_ONLY = 0x40,
+  CLEAR_ON_IGNITION_ON = 0x80,
+  BACKUP = 0x100,
+  ALL = 0xFFFFFFFF
+};
+
+enum ParamKeyType {
+  STRING = 0, // must be utf-8 decodable
+  BOOL = 1,
+  INT = 2,
+  FLOAT = 3,
+  TIME = 4, // ISO 8601
+  JSON = 5,
+  BYTES = 6
+};
+
+struct ParamKeyAttributes {
+  uint32_t flags;
+  ParamKeyType type;
+  std::optional<std::string> default_value = std::nullopt;
+};
+
+class Params {
+public:
+  explicit Params(const std::string &path = {});
+  ~Params();
+  // Not copyable.
+  Params(const Params&) = delete;
+  Params& operator=(const Params&) = delete;
+
+  std::vector<std::string> allKeys(ParamKeyFlag flag = ALL) const;
+  bool checkKey(const std::string &key);
+  ParamKeyFlag getKeyFlag(const std::string &key);
+  ParamKeyType getKeyType(const std::string &key);
+  std::optional<std::string> getKeyDefaultValue(const std::string &key);
+  inline std::string getParamPath(const std::string &key = {}) {
+    return params_path + params_prefix + (key.empty() ? "" : "/" + key);
+  }
+
+  // Delete a value
+  int remove(const std::string &key);
+  void clearAll(ParamKeyFlag flag);
+
+  // helpers for reading values
+  std::string get(const std::string &key, bool block = false);
+  inline bool getBool(const std::string &key, bool block = false) {
+    return get(key, block) == "1";
+  }
+  std::map<std::string, std::string> readAll();
+
+  // helpers for writing values
+  int put(const char *key, const char *val, size_t value_size);
+  inline int put(const std::string &key, const std::string &val) {
+    return put(key.c_str(), val.data(), val.size());
+  }
+  inline int putBool(const std::string &key, bool val) {
+    return put(key.c_str(), val ? "1" : "0", 1);
+  }
+  void putNonBlocking(const std::string &key, const std::string &val);
+  inline void putBoolNonBlocking(const std::string &key, bool val) {
+    putNonBlocking(key, val ? "1" : "0");
+  }
+
+private:
+  void asyncWriteThread();
+
+  std::string params_path;
+  std::string params_prefix;
+
+  // for nonblocking write
+  std::future<void> future;
+  SafeQueue<std::pair<std::string, std::string>> queue;
+};

openpilot/common/params.py

@@ -0,0 +1,19 @@
+from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
+assert Params
+assert ParamKeyFlag
+assert ParamKeyType
+assert UnknownKeyName
+
+if __name__ == "__main__":
+  import sys
+
+  params = Params()
+  key = sys.argv[1]
+  assert params.check_key(key), f"unknown param: {key}"
+
+  if len(sys.argv) == 3:
+    val = sys.argv[2]
+    print(f"SET: {key} = {val}")
+    params.put(key, val, block=True)
+  elif len(sys.argv) == 2:
+    print(f"GET: {key} = {params.get(key)}")

openpilot/common/params_keys.h

@@ -0,0 +1,280 @@
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
+
+inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
+    {"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
+    {"AdbEnabled", {PERSISTENT | BACKUP, BOOL}},
+    {"AlwaysOnDM", {PERSISTENT | BACKUP, BOOL}},
+    {"ApiCache_Device", {PERSISTENT, STRING}},
+    {"ApiCache_FirehoseStats", {PERSISTENT, JSON}},
+    {"AssistNowToken", {PERSISTENT, STRING}},
+    {"AthenadPid", {PERSISTENT, INT}},
+    {"AthenadUploadQueue", {PERSISTENT, JSON}},
+    {"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
+    {"BootCount", {PERSISTENT, INT}},
+    {"CalibrationParams", {PERSISTENT, BYTES}},
+    {"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
+    {"CarBatteryCapacity", {PERSISTENT, INT}},
+    {"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
+    {"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"CarParamsPersistent", {PERSISTENT, BYTES}},
+    {"CarParamsPrevRoute", {PERSISTENT, BYTES}},
+    {"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
+    {"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"CurrentBootlog", {PERSISTENT, STRING}},
+    {"CurrentRoute", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"DisableLogging", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"DisablePowerDown", {PERSISTENT | BACKUP, BOOL}},
+    {"DisableUpdates", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"DisengageOnAccelerator", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"DongleId", {PERSISTENT, STRING}},
+    {"DoReboot", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
+    {"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
+    {"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
+    {"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
+    {"FirmwareQueryDone", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ForcePowerDown", {PERSISTENT, BOOL}},
+    {"GitBranch", {PERSISTENT, STRING}},
+    {"GitCommit", {PERSISTENT, STRING}},
+    {"GitCommitDate", {PERSISTENT, STRING}},
+    {"GitDiff", {PERSISTENT, STRING}},
+    {"GithubSshKeys", {PERSISTENT | BACKUP, STRING}},
+    {"GithubUsername", {PERSISTENT | BACKUP, STRING}},
+    {"GitRemote", {PERSISTENT, STRING}},
+    {"GsmApn", {PERSISTENT | BACKUP, STRING}},
+    {"GsmMetered", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"GsmRoaming", {PERSISTENT | BACKUP, BOOL}},
+    {"HardwareSerial", {PERSISTENT, STRING}},
+    {"HasAcceptedTerms", {PERSISTENT, STRING, "0"}},
+    {"InstallDate", {PERSISTENT, TIME}},
+    {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsEngaged", {PERSISTENT, BOOL}},
+    {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
+    {"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
+    {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsRhdDetected", {PERSISTENT, BOOL}},
+    {"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
+    {"LastAthenaPingTime", {CLEAR_ON_MANAGER_START, INT}},
+    {"LastGPSPosition", {PERSISTENT, STRING}},
+    {"LastManagerExitReason", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastOffroadStatusPacket", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, JSON}},
+    {"LastAgnosPowerMonitorShutdown", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
+    {"LastUpdateRouteCount", {PERSISTENT, INT, "0"}},
+    {"LastUpdateTime", {PERSISTENT, TIME}},
+    {"LastUpdateUptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
+    {"LiveParameters", {PERSISTENT, JSON}},
+    {"LiveParametersV2", {PERSISTENT, BYTES}},
+    {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
+    {"LivestreamRequestKeyframe", {CLEAR_ON_MANAGER_START | DONT_LOG, BOOL}},
+    {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
+    {"LocationFilterInitialState", {PERSISTENT, BYTES}},
+    {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
+    {"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
+    {"ObdMultiplexingChanged", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"ObdMultiplexingEnabled", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"Offroad_CarUnrecognized", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
+    {"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_TemperatureTooHigh", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UnregisteredHardware", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_UpdateFailed", {CLEAR_ON_MANAGER_START, JSON}},
+    {"Offroad_DriverMonitoringUncertain", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"PrimeType", {PERSISTENT, INT}},
+    {"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
+    {"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RecordFront", {PERSISTENT | BACKUP, BOOL}},
+    {"RecordFrontLock", {PERSISTENT, BOOL}},  // for the internal fleet
+    {"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
+    {"ShowDebugInfo", {PERSISTENT, BOOL}},
+    {"RouteCount", {PERSISTENT, INT, "0"}},
+    {"SnoozeUpdate", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"SshEnabled", {PERSISTENT | BACKUP, BOOL}},
+    {"TermsVersion", {PERSISTENT, STRING}},
+    {"TorqueBar", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TrainingVersion", {PERSISTENT, STRING}},
+    {"UbloxAvailable", {PERSISTENT, BOOL}},
+    {"UpdateAvailable", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
+    {"UpdateFailedCount", {CLEAR_ON_MANAGER_START, INT}},
+    {"UpdaterAvailableBranches", {PERSISTENT, STRING}},
+    {"UpdaterCurrentDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterCurrentReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterFetchAvailable", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"UpdaterNewDescription", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterNewReleaseNotes", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"UpdaterState", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterTargetBranch", {CLEAR_ON_MANAGER_START, STRING}},
+    {"UpdaterLastFetchTime", {PERSISTENT, TIME}},
+    {"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
+    {"UsbGpuPresent", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"UsbGpuCompiled", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
+    {"Version", {PERSISTENT, STRING}},
+
+    // --- sunnypilot params --- //
+    {"ApiCache_DriveStats", {PERSISTENT, JSON}},
+    {"AutoLaneChangeBsmDelay", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"AutoLaneChangeTimer", {PERSISTENT | BACKUP, INT, "0"}},
+    {"BlinkerLateralReengageDelay", {PERSISTENT | BACKUP, INT, "0"}},  // seconds
+    {"BlinkerMinLateralControlSpeed", {PERSISTENT | BACKUP, INT, "20"}},  // MPH or km/h
+    {"BlinkerPauseLateralControl", {PERSISTENT | BACKUP, INT, "0"}},
+    {"Brightness", {PERSISTENT | BACKUP, INT, "0"}},
+    {"CarList", {PERSISTENT, JSON}},
+    {"CarParamsSP", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
+    {"CarParamsSPCache", {CLEAR_ON_MANAGER_START, BYTES}},
+    {"CarParamsSPPersistent", {PERSISTENT, BYTES}},
+    {"CarPlatformBundle", {PERSISTENT | BACKUP, JSON}},
+    {"ChevronInfo", {PERSISTENT | BACKUP, INT, "4"}},
+    {"CompletedSunnylinkConsentVersion", {PERSISTENT, STRING, "0"}},
+    {"CustomAccIncrementsEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"CustomAccLongPressIncrement", {PERSISTENT | BACKUP, INT, "5"}},
+    {"CustomAccShortPressIncrement", {PERSISTENT | BACKUP, INT, "1"}},
+    {"DeviceBootMode", {PERSISTENT | BACKUP, INT, "0"}},
+    {"DevUIInfo", {PERSISTENT | BACKUP, INT, "0"}},
+    {"EnableCopyparty", {PERSISTENT | BACKUP, BOOL}},
+    {"EnableGithubRunner", {PERSISTENT | BACKUP, BOOL}},
+    {"GreenLightAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"GithubRunnerSufficientVoltage", {CLEAR_ON_MANAGER_START , BOOL}},
+    {"HasAcceptedTermsSP", {PERSISTENT, STRING, "0"}},
+    {"HideVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"IntelligentCruiseButtonManagement", {PERSISTENT | BACKUP , BOOL}},
+    {"InteractivityTimeout", {PERSISTENT | BACKUP, INT, "0"}},
+    {"IsDevelopmentBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"IsReleaseSpBranch", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"LastGPSPositionLLK", {PERSISTENT, STRING}},
+    {"LeadDepartAlert", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}},
+    {"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}},
+    {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}},
+    {"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
+    {"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
+    {"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
+    {"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TrueVEgoUI", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // MADS params
+    {"Mads", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"MadsMainCruiseAllowed", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"MadsSteeringMode", {PERSISTENT | BACKUP, INT, "0"}},
+    {"MadsUnifiedEngagementMode", {PERSISTENT | BACKUP, BOOL, "1"}},
+
+    // Model Manager params
+    {"ModelManager_ActiveBundle", {PERSISTENT, JSON}},
+    {"ModelManager_ClearCache", {CLEAR_ON_MANAGER_START, BOOL}},
+    {"ModelManager_DownloadIndex", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, INT}},
+    {"ModelManager_Favs", {PERSISTENT | BACKUP, STRING}},
+    {"ModelManager_LastSyncTime", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, INT, "0"}},
+    {"ModelManager_ModelsCache", {PERSISTENT | BACKUP, JSON}},
+
+    // Neural Network Lateral Control
+    {"NeuralNetworkLateralControl", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // sunnylink params
+    {"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
+    {"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
+    {"ParamsVersion", {PERSISTENT, INT}},
+    {"SunnylinkCache_Roles", {PERSISTENT, STRING}},
+    {"SunnylinkCache_Users", {PERSISTENT, STRING}},
+    {"SunnylinkDongleId", {PERSISTENT, STRING}},
+    {"SunnylinkdPid", {PERSISTENT, INT}},
+    {"SunnylinkEnabled", {PERSISTENT, BOOL, "1"}},
+    {"SunnylinkTempFault", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL, "0"}},
+
+    // Backup Manager params
+    {"BackupManager_CreateBackup", {PERSISTENT, BOOL}},
+    {"BackupManager_RestoreVersion", {PERSISTENT, STRING}},
+
+    // sunnypilot car specific params
+    {"HyundaiLongitudinalTuning", {PERSISTENT | BACKUP, INT, "0"}},
+    {"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    {"DynamicExperimentalControl", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"BlindSpot", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // sunnypilot model params
+    {"CameraOffset", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
+    {"LagdToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
+    {"LagdToggleDelay", {PERSISTENT | BACKUP, FLOAT, "0.2"}},
+    {"LagdValueCache", {PERSISTENT, FLOAT, "0.2"}},
+    {"LaneTurnDesire", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"LaneTurnValue", {PERSISTENT | BACKUP, FLOAT, "19.0"}},
+    {"PlanplusControl", {PERSISTENT | BACKUP, FLOAT, "1.0"}},
+
+    // mapd
+    {"MapAdvisorySpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT}},
+    {"MapdVersion", {PERSISTENT, STRING}},
+    {"MapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, FLOAT, "0.0"}},
+    {"NextMapSpeedLimit", {CLEAR_ON_ONROAD_TRANSITION, JSON}},
+    {"Offroad_OSMUpdateRequired", {CLEAR_ON_MANAGER_START, JSON}},
+    {"OsmDbUpdatesCheck", {CLEAR_ON_MANAGER_START, BOOL}},  // mapd database update happens with device ON, reset on boot
+    {"OSMDownloadBounds", {PERSISTENT, STRING}},
+    {"OsmDownloadedDate", {PERSISTENT, STRING, "0.0"}},
+    {"OSMDownloadLocations", {PERSISTENT, JSON}},
+    {"OSMDownloadProgress", {CLEAR_ON_MANAGER_START, JSON}},
+    {"OsmLocal", {PERSISTENT, BOOL}},
+    {"OsmLocationName", {PERSISTENT, STRING}},
+    {"OsmLocationTitle", {PERSISTENT, STRING}},
+    {"OsmLocationUrl", {PERSISTENT, STRING}},
+    {"OsmStateName", {PERSISTENT, STRING, "All"}},
+    {"OsmStateTitle", {PERSISTENT, STRING}},
+    {"OsmWayTest", {PERSISTENT, STRING}},
+    {"RoadName", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"RoadNameToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // Speed Limit
+    {"SpeedLimitMode", {PERSISTENT | BACKUP, INT, "1"}},
+    {"SpeedLimitOffsetType", {PERSISTENT | BACKUP, INT, "0"}},
+    {"SpeedLimitPolicy", {PERSISTENT | BACKUP, INT, "3"}},
+    {"SpeedLimitValueOffset", {PERSISTENT | BACKUP, INT, "0"}},
+
+    // Smart Cruise Control
+    {"MapTargetVelocities", {CLEAR_ON_ONROAD_TRANSITION, STRING}},
+    {"SmartCruiseControlMap", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"SmartCruiseControlVision", {PERSISTENT | BACKUP, BOOL, "0"}},
+
+    // Torque lateral control custom params
+    {"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
+    {"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
+    {"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
+    {"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
+    {"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
+    {"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
+    {"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
+    {"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
+};

openpilot/common/params_pyx.pyx

@@ -0,0 +1,192 @@
+# distutils: language = c++
+# cython: language_level = 3
+import builtins
+import datetime
+import json
+from libcpp cimport bool
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.optional cimport optional
+
+from openpilot.common.swaglog import cloudlog
+
+cdef extern from "common/params.h":
+  cpdef enum ParamKeyFlag:
+    PERSISTENT
+    CLEAR_ON_MANAGER_START
+    CLEAR_ON_ONROAD_TRANSITION
+    CLEAR_ON_OFFROAD_TRANSITION
+    DEVELOPMENT_ONLY
+    CLEAR_ON_IGNITION_ON
+    BACKUP
+    ALL
+
+  cpdef enum ParamKeyType:
+    STRING
+    BOOL
+    INT
+    FLOAT
+    TIME
+    JSON
+    BYTES
+
+  cdef cppclass c_Params "Params":
+    c_Params(string) except + nogil
+    string get(string, bool) nogil
+    bool getBool(string, bool) nogil
+    int remove(string) nogil
+    int put(string, string) nogil
+    void putNonBlocking(string, string) nogil
+    void putBoolNonBlocking(string, bool) nogil
+    int putBool(string, bool) nogil
+    bool checkKey(string) nogil
+    ParamKeyType getKeyType(string) nogil
+    optional[string] getKeyDefaultValue(string) nogil
+    string getParamPath(string) nogil
+    void clearAll(ParamKeyFlag)
+    vector[string] allKeys(ParamKeyFlag)
+
+PYTHON_2_CPP = {
+  (str, STRING): lambda v: v,
+  (builtins.bool, BOOL): lambda v: "1" if v else "0",
+  (int, INT): str,
+  (float, FLOAT): str,
+  (datetime.datetime, TIME): lambda v: v.isoformat(),
+  (dict, JSON): json.dumps,
+  (list, JSON): json.dumps,
+  (bytes, BYTES): lambda v: v,
+}
+CPP_2_PYTHON = {
+  STRING: lambda v: v.decode("utf-8"),
+  BOOL: lambda v: v == b"1",
+  INT: int,
+  FLOAT: float,
+  TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
+  JSON: json.loads,
+  BYTES: lambda v: v,
+}
+
+def ensure_bytes(v):
+  return v.encode() if isinstance(v, str) else v
+
+class UnknownKeyName(Exception):
+  pass
+
+cdef class Params:
+  cdef c_Params* p
+  cdef str d
+
+  def __cinit__(self, d=""):
+    cdef string path = <string>d.encode()
+    with nogil:
+      self.p = new c_Params(path)
+    self.d = d
+
+  def __reduce__(self):
+    return (type(self), (self.d,))
+
+  def __dealloc__(self):
+    del self.p
+
+  def clear_all(self, tx_flag=ParamKeyFlag.ALL):
+    self.p.clearAll(tx_flag)
+
+  def check_key(self, key):
+    key = ensure_bytes(key)
+    if not self.p.checkKey(key):
+      raise UnknownKeyName(key)
+    return key
+
+  def python2cpp(self, proposed_type, expected_type, value, key):
+    cast = PYTHON_2_CPP.get((proposed_type, expected_type))
+    if cast:
+      return cast(value)
+    raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
+
+  def _cpp2python(self, t, value, default, key):
+    if value is None:
+      return None
+    try:
+      return CPP_2_PYTHON[t](value)
+    except (KeyError, TypeError, ValueError):
+      cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
+      return self._cpp2python(t, default, None, key)
+
+  def get(self, key, bool block=False, bool return_default=False):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    cdef string val
+    with nogil:
+      val = self.p.get(k, block)
+
+    default_val = (default.value() if default.has_value() else None) if return_default else None
+    if val == b"":
+      if block:
+        # If we got no value while running in blocked mode
+        # it means we got an interrupt while waiting
+        raise KeyboardInterrupt
+      else:
+        return self._cpp2python(t, default_val, None, key)
+    return self._cpp2python(t, val, default_val, key)
+
+  def get_bool(self, key, bool block=False):
+    cdef string k = self.check_key(key)
+    cdef bool r
+    with nogil:
+      r = self.p.getBool(k, block)
+    return r
+
+  def _put_cast(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
+
+  def put(self, key, dat, bool block = False):
+    """
+    Warning: block=True blocks until the param is written to disk!
+    In very rare cases this can take over a second, and your code will hang.
+    Use block=False in time sensitive code, but in general try to avoid
+    writing params as much as possible.
+    """
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = self._put_cast(key, dat)
+    with nogil:
+      if block:
+        self.p.put(k, dat_bytes)
+      else:
+        self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool(self, key, bool val, bool block = False):
+    cdef string k = self.check_key(key)
+    with nogil:
+      if block:
+        self.p.putBool(k, val)
+      else:
+        self.p.putBoolNonBlocking(k, val)
+
+  def remove(self, key):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.remove(k)
+
+  def get_param_path(self, key=""):
+    cdef string key_bytes = ensure_bytes(key)
+    return self.p.getParamPath(key_bytes).decode("utf-8")
+
+  def get_type(self, key):
+    return self.p.getKeyType(self.check_key(key))
+
+  def all_keys(self, flag=ParamKeyFlag.ALL):
+    return self.p.allKeys(flag)
+
+  def get_default_value(self, key):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    cdef optional[string] default = self.p.getKeyDefaultValue(k)
+    return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
+
+  def cpp2python(self, key, value):
+    cdef string k = self.check_key(key)
+    cdef ParamKeyType t = self.p.getKeyType(k)
+    return self._cpp2python(t, value, None, key)

openpilot/common/pid.py

@@ -0,0 +1,57 @@
+import numpy as np
+from numbers import Number
+
+class PIDController:
+  def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
+    self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
+    self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
+    self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
+
+    self.set_limits(pos_limit, neg_limit)
+
+    self.i_dt = 1.0 / rate
+    self.speed = 0.0
+
+    self.reset()
+
+  @property
+  def k_p(self):
+    return np.interp(self.speed, self._k_p[0], self._k_p[1])
+
+  @property
+  def k_i(self):
+    return np.interp(self.speed, self._k_i[0], self._k_i[1])
+
+  @property
+  def k_d(self):
+    return np.interp(self.speed, self._k_d[0], self._k_d[1])
+
+  def reset(self):
+    self.p = 0.0
+    self.i = 0.0
+    self.d = 0.0
+    self.f = 0.0
+    self.control = 0
+
+  def set_limits(self, pos_limit, neg_limit):
+    self.pos_limit = pos_limit
+    self.neg_limit = neg_limit
+
+  def update(self, error, error_rate=0.0, speed=0.0, feedforward=0., freeze_integrator=False):
+    self.speed = speed
+    self.p = self.k_p * float(error)
+    self.d = self.k_d * error_rate
+    self.f = feedforward
+
+    if not freeze_integrator:
+      i = self.i + self.k_i * self.i_dt * error
+
+      # Don't allow windup if already clipping
+      test_control = self.p + i + self.d + self.f
+      i_upperbound = self.i if test_control > self.pos_limit else self.pos_limit
+      i_lowerbound = self.i if test_control < self.neg_limit else self.neg_limit
+      self.i = np.clip(i, i_lowerbound, i_upperbound)
+
+    control = self.p + self.i + self.d + self.f
+    self.control = np.clip(control, self.neg_limit, self.pos_limit)
+    return self.control

openpilot/common/prefix.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <cassert>
+#include <string>
+
+#include "common/params.h"
+#include "common/util.h"
+#include "common/hardware/hw.h"
+
+class OpenpilotPrefix {
+public:
+  OpenpilotPrefix(std::string prefix = {}) {
+    if (prefix.empty()) {
+      prefix = util::random_string(15);
+    }
+#ifdef __APPLE__
+    msgq_path = "/tmp/msgq_" + prefix;
+#else
+    msgq_path = "/dev/shm/msgq_" + prefix;
+#endif
+    bool ret = util::create_directories(msgq_path, 0777);
+    assert(ret);
+    setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
+  }
+
+  ~OpenpilotPrefix() {
+    auto param_path = Params().getParamPath();
+    if (util::file_exists(param_path)) {
+      std::string real_path = util::readlink(param_path);
+      util::check_system(util::string_format("rm %s -rf", real_path.c_str()));
+      unlink(param_path.c_str());
+    }
+    if (getenv("COMMA_CACHE") == nullptr) {
+      util::check_system(util::string_format("rm %s -rf", Path::download_cache_root().c_str()));
+    }
+    util::check_system(util::string_format("rm %s -rf", Path::comma_home().c_str()));
+    util::check_system(util::string_format("rm %s -rf", msgq_path.c_str()));
+    unsetenv("OPENPILOT_PREFIX");
+  }
+
+private:
+  std::string msgq_path;
+};

openpilot/common/prefix.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import shutil
+import uuid
+
+
+from openpilot.common.params import Params
+from openpilot.common.hardware import PC
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+
+class OpenpilotPrefix:
+  def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+    self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
+    shm_path = "/tmp" if platform.system() == "Darwin" else "/dev/shm"
+    self.msgq_path = os.path.join(shm_path, "msgq_" + self.prefix)
+    self.create_dirs_on_enter = create_dirs_on_enter
+    self.clean_dirs_on_exit = clean_dirs_on_exit
+    self.shared_download_cache = shared_download_cache
+
+  def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
+    os.environ['OPENPILOT_PREFIX'] = self.prefix
+
+    if self.create_dirs_on_enter:
+      self.create_dirs()
+
+    if self.shared_download_cache:
+      os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    if self.clean_dirs_on_exit:
+      self.clean_dirs()
+    try:
+      del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
+    except KeyError:
+      pass
+    return False
+
+  def create_dirs(self):
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
+
+  def clean_dirs(self):
+    symlink_path = Params().get_param_path()
+    if os.path.exists(symlink_path):
+      shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
+      os.remove(symlink_path)
+    shutil.rmtree(self.msgq_path, ignore_errors=True)
+    if PC:
+      shutil.rmtree(Paths.log_root(), ignore_errors=True)
+    if not os.environ.get("COMMA_CACHE", False):
+      shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
+    shutil.rmtree(Paths.comma_home(), ignore_errors=True)

openpilot/common/queue.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+
+template <class T>
+class SafeQueue {
+public:
+  SafeQueue() = default;
+
+  void push(const T& v) {
+    {
+      std::unique_lock lk(m);
+      q.push(v);
+    }
+    cv.notify_one();
+  }
+
+  T pop() {
+    std::unique_lock lk(m);
+    cv.wait(lk, [this] { return !q.empty(); });
+    T v = q.front();
+    q.pop();
+    return v;
+  }
+
+  bool try_pop(T& v, int timeout_ms = 0) {
+    std::unique_lock lk(m);
+    if (!cv.wait_for(lk, std::chrono::milliseconds(timeout_ms), [this] { return !q.empty(); })) {
+      return false;
+    }
+    v = q.front();
+    q.pop();
+    return true;
+  }
+
+  bool empty() const {
+    std::scoped_lock lk(m);
+    return q.empty();
+  }
+
+  size_t size() const {
+    std::scoped_lock lk(m);
+    return q.size();
+  }
+
+private:
+  mutable std::mutex m;
+  std::condition_variable cv;
+  std::queue<T> q;
+};

openpilot/common/ratekeeper.cc

@@ -0,0 +1,40 @@
+#include "common/ratekeeper.h"
+
+#include <algorithm>
+
+#include "common/swaglog.h"
+#include "common/timing.h"
+#include "common/util.h"
+
+RateKeeper::RateKeeper(const std::string &name_, float rate, float print_delay_threshold_)
+    : name(name_),
+      print_delay_threshold(std::max(0.f, print_delay_threshold_)) {
+  interval = 1 / rate;
+  last_monitor_time = seconds_since_boot();
+  next_frame_time = last_monitor_time + interval;
+}
+
+bool RateKeeper::keepTime() {
+  bool lagged = monitorTime();
+  if (remaining_ > 0) {
+    util::sleep_for(remaining_ * 1000);
+  }
+  return lagged;
+}
+
+bool RateKeeper::monitorTime() {
+  ++frame_;
+  last_monitor_time = seconds_since_boot();
+  remaining_ = next_frame_time - last_monitor_time;
+
+  bool lagged = remaining_ < 0;
+  if (lagged) {
+    if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
+      LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
+    }
+    next_frame_time = last_monitor_time + interval;
+  } else {
+    next_frame_time += interval;
+  }
+  return lagged;
+}

openpilot/common/ratekeeper.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+class RateKeeper {
+public:
+  RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
+  ~RateKeeper() {}
+  bool keepTime();
+  bool monitorTime();
+  inline uint64_t frame() const { return frame_; }
+  inline double remaining() const { return remaining_; }
+
+private:
+  double interval;
+  double next_frame_time;
+  double last_monitor_time;
+  double remaining_ = 0;
+  float print_delay_threshold = 0;
+  uint64_t frame_ = 0;
+  std::string name;
+};

openpilot/common/realtime.py

@@ -0,0 +1,101 @@
+"""Utilities for reading real time clocks and keeping soft real time constraints."""
+import gc
+import os
+import sys
+import time
+
+from setproctitle import getproctitle
+
+from openpilot.common.utils import MovingAverage
+from openpilot.common.hardware import PC
+
+
+# time step for each process
+DT_CTRL = 0.01  # controlsd
+DT_MDL = 0.05  # model
+DT_HW = 0.5  # hardwared and manager
+DT_DMON = 0.05  # driver monitoring
+
+
+class Priority:
+  # CORE 2
+  # - modeld = 55
+  # - camerad = 54
+  CTRL_LOW = 51 # plannerd & radard
+
+  # CORE 3
+  # - pandad = 55
+  CTRL_HIGH = 53
+
+
+def drop_realtime() -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
+
+
+def set_core_affinity(cores: list[int]) -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setaffinity(0, cores)
+
+
+def config_realtime_process(cores: int | list[int], priority: int) -> None:
+  gc.disable()
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
+  c = cores if isinstance(cores, list) else [cores, ]
+  set_core_affinity(c)
+
+
+class Ratekeeper:
+  def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
+    """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
+    self._interval = 1. / rate
+    self._print_delay_threshold = print_delay_threshold
+    self._frame = 0
+    self._remaining = 0.0
+    self._process_name = getproctitle()
+    self._last_monitor_time = -1.
+    self._next_frame_time = -1.
+
+    self.avg_dt = MovingAverage(100)
+    self.avg_dt.add_value(self._interval)
+
+  @property
+  def frame(self) -> int:
+    return self._frame
+
+  @property
+  def remaining(self) -> float:
+    return self._remaining
+
+  @property
+  def lagging(self) -> bool:
+    expected_dt = self._interval * (1 / 0.9)
+    return self.avg_dt.get_average() > expected_dt
+
+  # Maintain loop rate by calling this at the end of each loop
+  def keep_time(self) -> bool:
+    lagged = self.monitor_time()
+    if self._remaining > 0:
+      time.sleep(self._remaining)
+    return lagged
+
+  # Monitors the cumulative lag, but does not enforce a rate
+  def monitor_time(self) -> bool:
+    if self._last_monitor_time < 0:
+      self._next_frame_time = time.monotonic() + self._interval
+      self._last_monitor_time = time.monotonic()
+
+    prev = self._last_monitor_time
+    self._last_monitor_time = time.monotonic()
+    self.avg_dt.add_value(self._last_monitor_time - prev)
+
+    lagged = False
+    remaining = self._next_frame_time - time.monotonic()
+    self._next_frame_time += self._interval
+    if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
+      print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
+      lagged = True
+    self._frame += 1
+    self._remaining = remaining
+    return lagged

openpilot/common/serial.py

@@ -0,0 +1,272 @@
+import errno
+import fcntl
+import os
+import select
+import struct
+import termios
+import time
+
+
+# Modem control lines (linux/termios.h); fall back to common x86_64 values.
+TIOCMBIS = getattr(termios, "TIOCMBIS", 0x5416)
+TIOCMBIC = getattr(termios, "TIOCMBIC", 0x5417)
+TIOCM_DTR = getattr(termios, "TIOCM_DTR", 0x002)
+TIOCM_RTS = getattr(termios, "TIOCM_RTS", 0x004)
+_TIOCM_DTR = struct.pack("I", TIOCM_DTR)
+_TIOCM_RTS = struct.pack("I", TIOCM_RTS)
+
+
+class SerialException(OSError):
+  pass
+
+
+class Serial:
+  def __init__(self, port: str, baudrate: int = 9600, timeout: float | None = None, *,
+               rtscts: bool = False, dsrdtr: bool = False, exclusive: bool = False):
+    self._port = port
+    self._baudrate = baudrate
+    self._timeout = timeout
+    self._rtscts = rtscts
+    self._dsrdtr = dsrdtr
+    self._exclusive = exclusive
+    self._dtr = True
+    self._fd = -1
+    self.open()
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *args) -> None:
+    self.close()
+
+  @property
+  def fd(self) -> int:
+    self._ensure_open()
+    return self._fd
+
+  @property
+  def baudrate(self) -> int:
+    return self._baudrate
+
+  @baudrate.setter
+  def baudrate(self, value: int) -> None:
+    self._baudrate = int(value)
+    if self._fd >= 0:
+      self._configure()
+
+  @property
+  def dtr(self) -> bool:
+    return self._dtr
+
+  @dtr.setter
+  def dtr(self, value: bool) -> None:
+    self._dtr = bool(value)
+    if self._fd >= 0:
+      self._set_line(TIOCM_DTR, _TIOCM_DTR, self._dtr)
+
+  def open(self) -> None:
+    if self._fd >= 0:
+      return
+    try:
+      self._fd = os.open(self._port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+    except OSError as e:
+      self._fd = -1
+      raise SerialException(e.errno, f"could not open port {self._port}: {e}") from e
+
+    try:
+      if self._exclusive:
+        try:
+          fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError as e:
+          raise SerialException(e.errno, f"could not exclusively lock port {self._port}: {e}") from e
+
+      self._configure()
+
+      # When not using hardware DSR/DTR handshaking, drive lines ourselves.
+      if not self._dsrdtr:
+        try:
+          self._set_line(TIOCM_DTR, _TIOCM_DTR, self._dtr)
+          if not self._rtscts:
+            self._set_line(TIOCM_RTS, _TIOCM_RTS, True)
+        except OSError as e:
+          if e.errno not in (errno.EINVAL, errno.ENOTTY):
+            raise
+
+      self.reset_input_buffer()
+    except BaseException:
+      self._close_fd()
+      raise
+
+  def close(self) -> None:
+    self._close_fd()
+
+  def read(self, size: int = 1) -> bytes:
+    self._ensure_open()
+    if size <= 0:
+      return b""
+
+    buf = bytearray()
+    deadline = self._deadline()
+    while len(buf) < size:
+      remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
+      if not self._wait_readable(remaining):
+        break
+      try:
+        chunk = os.read(self._fd, size - len(buf))
+      except InterruptedError:
+        continue
+      except OSError as e:
+        if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
+          if self._timeout == 0:
+            break
+          continue
+        raise SerialException(e.errno, f"read failed: {e}") from e
+      if not chunk:
+        break
+      buf.extend(chunk)
+    return bytes(buf)
+
+  def readline(self) -> bytes:
+    self._ensure_open()
+    buf = bytearray()
+    deadline = self._deadline()
+    while True:
+      remaining = None if deadline is None else max(0.0, deadline - time.monotonic())
+      if deadline is not None and remaining == 0.0 and not buf:
+        # match pyserial: timed-out readline returns empty
+        if not self._wait_readable(0.0):
+          return b""
+      elif not self._wait_readable(remaining):
+        return bytes(buf)
+
+      try:
+        chunk = os.read(self._fd, 1)
+      except InterruptedError:
+        continue
+      except OSError as e:
+        if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
+          if self._timeout == 0:
+            return bytes(buf)
+          continue
+        raise SerialException(e.errno, f"read failed: {e}") from e
+      if not chunk:
+        return bytes(buf)
+      buf.extend(chunk)
+      if chunk == b"\n":
+        return bytes(buf)
+
+  def write(self, data: bytes) -> int:
+    self._ensure_open()
+    if not data:
+      return 0
+    view = memoryview(data)
+    total = 0
+    while total < len(data):
+      try:
+        n = os.write(self._fd, view[total:])
+      except InterruptedError:
+        continue
+      except OSError as e:
+        if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK):
+          select.select([], [self._fd], [], None)
+          continue
+        raise SerialException(e.errno, f"write failed: {e}") from e
+      if n == 0:
+        raise SerialException("write returned 0")
+      total += n
+    return total
+
+  def flush(self) -> None:
+    self._ensure_open()
+    termios.tcdrain(self._fd)
+
+  def reset_input_buffer(self) -> None:
+    self._ensure_open()
+    termios.tcflush(self._fd, termios.TCIFLUSH)
+
+  def reset_output_buffer(self) -> None:
+    self._ensure_open()
+    termios.tcflush(self._fd, termios.TCOFLUSH)
+
+  def _close_fd(self) -> None:
+    if self._fd >= 0:
+      try:
+        if self._exclusive:
+          fcntl.flock(self._fd, fcntl.LOCK_UN)
+      except OSError:
+        pass
+      try:
+        os.close(self._fd)
+      except OSError:
+        pass
+      self._fd = -1
+
+  def _ensure_open(self) -> None:
+    if self._fd < 0:
+      raise SerialException("port is not open")
+
+  def _deadline(self) -> float | None:
+    if self._timeout is None:
+      return None
+    if self._timeout == 0:
+      return time.monotonic()
+    return time.monotonic() + self._timeout
+
+  def _wait_readable(self, timeout: float | None) -> bool:
+    """Return True if fd is readable. timeout None blocks; 0 polls."""
+    if timeout is not None and timeout < 0:
+      timeout = 0.0
+    try:
+      ready, _, _ = select.select([self._fd], [], [], timeout)
+    except InterruptedError:
+      return False
+    return bool(ready)
+
+  def _baud_constant(self, baudrate: int) -> int:
+    try:
+      return getattr(termios, f"B{baudrate}")
+    except AttributeError as e:
+      raise ValueError(f"unsupported baud rate: {baudrate}") from e
+
+  def _configure(self) -> None:
+    self._ensure_open()
+    try:
+      attrs = termios.tcgetattr(self._fd)
+    except termios.error as e:
+      raise SerialException(f"could not get port attributes: {e}") from e
+
+    iflag, oflag, cflag, lflag, _ispeed, _ospeed, cc = attrs
+
+    # raw binary 8N1
+    iflag = 0
+    oflag = 0
+    lflag = 0
+    cflag |= termios.CLOCAL | termios.CREAD
+    cflag &= ~termios.CSIZE
+    cflag |= termios.CS8
+    cflag &= ~(termios.PARENB | termios.PARODD | termios.CSTOPB)
+
+    if hasattr(termios, "CRTSCTS"):
+      if self._rtscts:
+        cflag |= termios.CRTSCTS
+      else:
+        cflag &= ~termios.CRTSCTS
+
+    speed = self._baud_constant(self._baudrate)
+    cc = list(cc)
+    # Non-blocking reads are handled via select + O_NONBLOCK; keep VMIN/VTIME at 0.
+    cc[termios.VMIN] = 0
+    cc[termios.VTIME] = 0
+
+    try:
+      termios.tcsetattr(self._fd, termios.TCSANOW, [iflag, oflag, cflag, lflag, speed, speed, cc])
+    except termios.error as e:
+      raise SerialException(f"could not configure port: {e}") from e
+
+    # Keep the fd non-blocking so timeout=0 and select work consistently.
+    flags = fcntl.fcntl(self._fd, fcntl.F_GETFL)
+    fcntl.fcntl(self._fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+  def _set_line(self, _bit: int, packed: bytes, enabled: bool) -> None:
+    request = TIOCMBIS if enabled else TIOCMBIC
+    fcntl.ioctl(self._fd, request, packed)

openpilot/common/simple_kalman.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+
+def get_kalman_gain(dt, A, C, Q, R, iterations=100):
+  P = np.zeros_like(Q)
+  for _ in range(iterations):
+    P = A.dot(P).dot(A.T) + dt * Q
+    S = C.dot(P).dot(C.T) + R
+    K = P.dot(C.T).dot(np.linalg.inv(S))
+    P = (np.eye(len(P)) - K.dot(C)).dot(P)
+  return K
+
+
+class KF1D:
+  # this EKF assumes constant covariance matrix, so calculations are much simpler
+  # the Kalman gain also needs to be precomputed using the control module
+
+  def __init__(self, x0, A, C, K):
+    self.x0_0 = x0[0][0]
+    self.x1_0 = x0[1][0]
+    self.A0_0 = A[0][0]
+    self.A0_1 = A[0][1]
+    self.A1_0 = A[1][0]
+    self.A1_1 = A[1][1]
+    self.C0_0 = C[0]
+    self.C0_1 = C[1]
+    self.K0_0 = K[0][0]
+    self.K1_0 = K[1][0]
+
+    self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
+    self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
+    self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
+    self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
+
+    # K matrix needs to  be pre-computed as follow:
+    # import control
+    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
+    # self.K = np.transpose(K)
+
+  def update(self, meas):
+    #self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
+    x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
+    x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
+    self.x0_0 = x0_0
+    self.x1_0 = x1_0
+    return [self.x0_0, self.x1_0]
+
+  @property
+  def x(self):
+    return [[self.x0_0], [self.x1_0]]
+
+  def set_x(self, x):
+    self.x0_0 = x[0][0]
+    self.x1_0 = x[1][0]

openpilot/common/spinner.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+from openpilot.common.basedir import BASEDIR
+
+
+class Spinner:
+  def __init__(self):
+    try:
+      self.spinner_proc = subprocess.Popen(["./spinner.py"],
+                                           stdin=subprocess.PIPE,
+                                           cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
+                                           close_fds=True)
+    except OSError:
+      self.spinner_proc = None
+
+  def __enter__(self):
+    return self
+
+  def update(self, spinner_text: str):
+    if self.spinner_proc is not None:
+      self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
+      try:
+        self.spinner_proc.stdin.flush()
+      except BrokenPipeError:
+        pass
+
+  def update_progress(self, cur: float, total: float):
+    self.update(str(round(100 * cur / total)))
+
+  def close(self):
+    if self.spinner_proc is not None:
+      self.spinner_proc.kill()
+      try:
+        self.spinner_proc.communicate(timeout=2.)
+      except subprocess.TimeoutExpired:
+        print("WARNING: failed to kill spinner")
+      self.spinner_proc = None
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+if __name__ == "__main__":
+  import time
+  with Spinner() as s:
+    s.update("Spinner text")
+    time.sleep(5.0)
+  print("gone")
+  time.sleep(5.0)

openpilot/common/stat_live.py

@@ -0,0 +1,73 @@
+import numpy as np
+
+class RunningStat:
+  # tracks realtime mean and standard deviation without storing any data
+  def __init__(self, priors=None, max_trackable=-1):
+    self.max_trackable = max_trackable
+    if priors is not None:
+      # initialize from history
+      self.M = priors[0]
+      self.S = priors[1]
+      self.n = priors[2]
+      self.M_last = self.M
+      self.S_last = self.S
+
+    else:
+      self.reset()
+
+  def reset(self):
+    self.M = 0.
+    self.S = 0.
+    self.M_last = 0.
+    self.S_last = 0.
+    self.n = 0
+
+  def push_data(self, new_data):
+    # short term memory hack
+    if self.max_trackable < 0 or self.n < self.max_trackable:
+      self.n += 1
+    if self.n == 0:
+      self.M_last = new_data
+      self.M = self.M_last
+      self.S_last = 0.
+    else:
+      self.M = self.M_last + (new_data - self.M_last) / self.n
+      self.S = self.S_last + (new_data - self.M_last) * (new_data - self.M)
+      self.M_last = self.M
+      self.S_last = self.S
+
+  def mean(self):
+    return self.M
+
+  def variance(self):
+    if self.n >= 2:
+      return self.S / (self.n - 1.)
+    else:
+      return 0
+
+  def std(self):
+    return np.sqrt(self.variance())
+
+  def params_to_save(self):
+    return [self.M, self.S, self.n]
+
+class RunningStatFilter:
+  def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
+    self.raw_stat = RunningStat(raw_priors, -1)
+    self.filtered_stat = RunningStat(filtered_priors, max_trackable)
+
+  def reset(self):
+    self.raw_stat.reset()
+    self.filtered_stat.reset()
+
+  def push_and_update(self, new_data):
+    _std_last = self.raw_stat.std()
+    self.raw_stat.push_data(new_data)
+    _delta_std = self.raw_stat.std() - _std_last
+    if _delta_std <= 0:
+      self.filtered_stat.push_data(new_data)
+    else:
+      pass
+      # self.filtered_stat.push_data(self.filtered_stat.mean())
+
+# class SequentialBayesian():

openpilot/common/swaglog.cc

@@ -0,0 +1,157 @@
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include "common/swaglog.h"
+
+#include <cassert>
+#include <limits>
+#include <mutex>
+#include <string>
+
+#include <zmq.h>
+#include <stdarg.h>
+#include "json11/json11.hpp"
+#include "common/version.h"
+#include "common/hardware/hw.h"
+
+#include "sunnypilot/common/version.h"
+
+class SwaglogState {
+public:
+  SwaglogState() {
+    zctx = zmq_ctx_new();
+    sock = zmq_socket(zctx, ZMQ_PUSH);
+
+    // Timeout on shutdown for messages to be received by the logging process
+    int timeout = 100;
+    zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
+    zmq_connect(sock, Path::swaglog_ipc().c_str());
+
+    // workaround for https://github.com/dropbox/json11/issues/38
+    setlocale(LC_NUMERIC, "C");
+
+    print_level = CLOUDLOG_WARNING;
+    if (const char* print_lvl = getenv("LOGPRINT")) {
+      if (strcmp(print_lvl, "debug") == 0) {
+        print_level = CLOUDLOG_DEBUG;
+      } else if (strcmp(print_lvl, "info") == 0) {
+        print_level = CLOUDLOG_INFO;
+      } else if (strcmp(print_lvl, "warning") == 0) {
+        print_level = CLOUDLOG_WARNING;
+      }
+    }
+
+    ctx_j = json11::Json::object{};
+    if (char* dongle_id = getenv("DONGLE_ID")) {
+      ctx_j["dongle_id"] = dongle_id;
+    }
+    if (char* git_origin = getenv("GIT_ORIGIN")) {
+      ctx_j["origin"] = git_origin;
+    }
+    if (char* git_branch = getenv("GIT_BRANCH")) {
+      ctx_j["branch"] = git_branch;
+    }
+    if (char* git_commit = getenv("GIT_COMMIT")) {
+      ctx_j["commit"] = git_commit;
+    }
+    if (char* daemon_name = getenv("MANAGER_DAEMON")) {
+      ctx_j["daemon"] = daemon_name;
+    }
+    ctx_j["version"] = SUNNYPILOT_VERSION;
+    ctx_j["dirty"] = !getenv("CLEAN");
+    ctx_j["device"] = Hardware::get_name();
+  }
+
+  ~SwaglogState() {
+    zmq_close(sock);
+    zmq_ctx_destroy(zctx);
+  }
+
+  void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
+    std::lock_guard lk(lock);
+    if (levelnum >= print_level) {
+      printf("%s: %s\n", filename, msg);
+    }
+    zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
+  }
+
+  std::mutex lock;
+  void* zctx = nullptr;
+  void* sock = nullptr;
+  int print_level;
+  json11::Json::object ctx_j;
+};
+
+bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
+uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
+
+static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
+                            char* msg_buf, const json11::Json::object &msg_j={}) {
+  static SwaglogState s;
+
+  json11::Json::object log_j = json11::Json::object {
+    {"ctx", s.ctx_j},
+    {"levelnum", levelnum},
+    {"filename", filename},
+    {"lineno", lineno},
+    {"funcname", func},
+    {"created", seconds_since_epoch()}
+  };
+  if (msg_j.empty()) {
+    log_j["msg"] = msg_buf;
+  } else {
+    log_j["msg"] = msg_j;
+  }
+
+  std::string log_s;
+  log_s += (char)levelnum;
+  ((json11::Json)log_j).dump(log_s);
+  s.log(levelnum, filename, lineno, func, msg_buf, log_s);
+
+  free(msg_buf);
+}
+
+void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
+                const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  char* msg_buf = nullptr;
+  int ret = vasprintf(&msg_buf, fmt, args);
+  va_end(args);
+  if (ret <= 0 || !msg_buf) return;
+  cloudlog_common(levelnum, filename, lineno, func, msg_buf);
+}
+
+void cloudlog_t_common(int levelnum, const char* filename, int lineno, const char* func,
+                       uint32_t frame_id, const char* fmt, va_list args) {
+  if (!LOG_TIMESTAMPS) return;
+  char* msg_buf = nullptr;
+  int ret = vasprintf(&msg_buf, fmt, args);
+  if (ret <= 0 || !msg_buf) return;
+  json11::Json::object tspt_j = json11::Json::object{
+    {"event", msg_buf},
+    {"time", std::to_string(nanos_since_boot())}
+  };
+  if (frame_id < NO_FRAME_ID) {
+    tspt_j["frame_id"] = std::to_string(frame_id);
+  }
+  tspt_j = json11::Json::object{{"timestamp", tspt_j}};
+  cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
+}
+
+
+void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
+                 const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  cloudlog_t_common(levelnum, filename, lineno, func, NO_FRAME_ID, fmt, args);
+  va_end(args);
+}
+void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
+                 uint32_t frame_id, const char* fmt, ...) {
+  va_list args;
+  va_start(args, fmt);
+  cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
+  va_end(args);
+}

openpilot/common/swaglog.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include "common/timing.h"
+
+#define CLOUDLOG_DEBUG 10
+#define CLOUDLOG_INFO 20
+#define CLOUDLOG_WARNING 30
+#define CLOUDLOG_ERROR 40
+#define CLOUDLOG_CRITICAL 50
+
+
+#ifdef __GNUC__
+#define SWAG_LOG_CHECK_FMT(a, b) __attribute__ ((format (printf, a, b)))
+#else
+#define SWAG_LOG_CHECK_FMT(a, b)
+#endif
+
+void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
+                const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
+
+void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
+                 const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
+
+void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
+                 uint32_t frame_id, const char* fmt, ...) SWAG_LOG_CHECK_FMT(6, 7);
+
+
+#define cloudlog(lvl, fmt, ...) cloudlog_e(lvl, __FILE__, __LINE__, \
+                                           __func__, \
+                                           fmt, ## __VA_ARGS__)
+
+#define cloudlog_t(lvl, ...) cloudlog_te(lvl, __FILE__, __LINE__, \
+                                          __func__, \
+                                          __VA_ARGS__)
+
+
+#define cloudlog_rl(burst, millis, lvl, fmt, ...)   \
+{                                                   \
+  static uint64_t __begin = 0;                      \
+  static int __printed = 0;                         \
+  static int __missed = 0;                          \
+                                                    \
+  int __burst = (burst);                            \
+  int __millis = (millis);                          \
+  uint64_t __ts = nanos_since_boot();               \
+                                                    \
+  if (!__begin) { __begin = __ts; }                 \
+                                                    \
+  if (__begin + __millis*1000000ULL < __ts) {       \
+    if (__missed) {                                 \
+      cloudlog(CLOUDLOG_WARNING, "cloudlog: %d messages suppressed", __missed); \
+    }                                               \
+    __begin = 0;                                    \
+    __printed = 0;                                  \
+    __missed = 0;                                   \
+  }                                                 \
+                                                    \
+  if (__printed < __burst) {                        \
+    cloudlog(lvl, fmt, ## __VA_ARGS__);             \
+    __printed++;                                    \
+  } else {                                          \
+    __missed++;                                     \
+  }                                                 \
+}
+
+
+#define LOGT(...) cloudlog_t(CLOUDLOG_DEBUG, __VA_ARGS__)
+#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
+#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
+#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
+#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
+
+#define LOGD_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
+#define LOG_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
+#define LOGW_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
+#define LOGE_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)

openpilot/common/swaglog.py

@@ -0,0 +1,144 @@
+import logging
+import os
+import time
+import warnings
+from pathlib import Path
+from logging.handlers import BaseRotatingHandler
+
+import zmq
+
+from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
+from openpilot.common.hardware.hw import Paths
+
+
+def get_file_handler():
+  Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
+  base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
+  handler = SwaglogRotatingFileHandler(base_filename)
+  return handler
+
+class SwaglogRotatingFileHandler(BaseRotatingHandler):
+  def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
+    super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
+    self.base_filename = base_filename
+    self.interval = interval # seconds
+    self.max_bytes = max_bytes
+    self.backup_count = backup_count
+    self.log_files = self.get_existing_logfiles()
+    log_indexes = [f.split(".")[-1] for f in self.log_files]
+    self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
+    self.last_rollover = None
+    self.doRollover()
+
+  def _open(self):
+    self.last_rollover = time.monotonic()
+    self.last_file_idx += 1
+    next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
+    stream = open(next_filename, self.mode, encoding=self.encoding)
+    self.log_files.insert(0, next_filename)
+    return stream
+
+  def get_existing_logfiles(self):
+    log_files = list()
+    base_dir = os.path.dirname(self.base_filename)
+    for fn in os.listdir(base_dir):
+      fp = os.path.join(base_dir, fn)
+      if fp.startswith(self.base_filename) and os.path.isfile(fp):
+        log_files.append(fp)
+    return sorted(log_files)
+
+  def shouldRollover(self, record):
+    size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
+    time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
+    return size_exceeded or time_exceeded
+
+  def doRollover(self):
+    if self.stream:
+      self.stream.close()
+    self.stream = self._open()
+
+    if self.backup_count > 0:
+      while len(self.log_files) > self.backup_count:
+        to_delete = self.log_files.pop()
+        if os.path.exists(to_delete): # just being safe, should always exist
+          os.remove(to_delete)
+
+class UnixDomainSocketHandler(logging.Handler):
+  def __init__(self, formatter):
+    logging.Handler.__init__(self)
+    self.setFormatter(formatter)
+    self.pid = None
+
+    self.zctx = None
+    self.sock = None
+
+  def __del__(self):
+    self.close()
+
+  def close(self):
+    if self.sock is not None:
+      self.sock.close()
+    if self.zctx is not None:
+      self.zctx.term()
+
+  def connect(self):
+    self.zctx = zmq.Context()
+    self.sock = self.zctx.socket(zmq.PUSH)
+    self.sock.setsockopt(zmq.LINGER, 10)
+    self.sock.connect(Paths.swaglog_ipc())
+    self.pid = os.getpid()
+
+  def emit(self, record):
+    if os.getpid() != self.pid:
+      # TODO suppresses warning about forking proc with zmq socket, fix root cause
+      warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
+      self.connect()
+
+    msg = self.format(record).rstrip('\n')
+    # print("SEND".format(repr(msg)))
+    try:
+      s = chr(record.levelno)+msg
+      self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
+    except zmq.error.Again:
+      # drop :/
+      pass
+
+
+class ForwardingHandler(logging.Handler):
+  def __init__(self, target_logger):
+    super().__init__()
+    self.target_logger = target_logger
+
+  def emit(self, record):
+    self.target_logger.handle(record)
+
+
+def add_file_handler(log):
+  """
+  Function to add the file log handler to swaglog.
+  This can be used to store logs when logmessaged is not running.
+  """
+  handler = get_file_handler()
+  handler.setFormatter(SwagLogFileFormatter(log))
+  log.addHandler(handler)
+
+
+cloudlog = log = SwagLogger()
+log.setLevel(logging.DEBUG)
+
+
+outhandler = logging.StreamHandler()
+
+print_level = os.environ.get('LOGPRINT', 'warning')
+if print_level == 'debug':
+  outhandler.setLevel(logging.DEBUG)
+elif print_level == 'info':
+  outhandler.setLevel(logging.INFO)
+elif print_level == 'warning':
+  outhandler.setLevel(logging.WARNING)
+
+ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
+
+log.addHandler(outhandler)
+# logs are sent through IPC before writing to disk to prevent disk I/O blocking
+log.addHandler(ipchandler)

openpilot/common/tests/.gitignore

@@ -0,0 +1 @@
+test_common

openpilot/common/tests/test_file_helpers.py

@@ -0,0 +1,19 @@
+import os
+from uuid import uuid4
+
+from openpilot.common.utils import atomic_write
+
+
+class TestFileHelpers:
+  def run_atomic_write_func(self, atomic_write_func):
+    path = f"/tmp/tmp{uuid4()}"
+    with atomic_write_func(path) as f:
+      f.write("test")
+      assert not os.path.exists(path)
+
+    with open(path) as f:
+      assert f.read() == "test"
+    os.remove(path)
+
+  def test_atomic_write(self):
+    self.run_atomic_write_func(atomic_write)

openpilot/common/tests/test_markdown.py

@@ -0,0 +1,15 @@
+import os
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.markdown import parse_markdown
+
+
+class TestMarkdown:
+  def test_all_release_notes(self):
+    with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
+      release_notes = f.read().split("\n\n")
+      assert len(release_notes) > 10
+
+      for rn in release_notes:
+        md = parse_markdown(rn)
+        assert len(md) > 0

openpilot/common/tests/test_params.py

@@ -0,0 +1,141 @@
+import pytest
+import datetime
+import os
+import threading
+import time
+import uuid
+
+from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
+
+class TestParams:
+  def setup_method(self):
+    self.params = Params()
+
+  def test_params_put_and_get(self):
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
+    assert self.params.get("DongleId") == "cb38263377b873ee"
+
+  def test_params_non_ascii(self):
+    st = b"\xe1\x90\xff"
+    self.params.put("CarParams", st, block=True)
+    assert self.params.get("CarParams") == st
+
+  def test_params_get_cleared_manager_start(self):
+    self.params.put("CarParams", b"test", block=True)
+    self.params.put("DongleId", "cb38263377b873ee", block=True)
+    assert self.params.get("CarParams") == b"test"
+
+    undefined_param = self.params.get_param_path(uuid.uuid4().hex)
+    with open(undefined_param, "w") as f:
+      f.write("test")
+    assert os.path.isfile(undefined_param)
+
+    self.params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
+    assert self.params.get("CarParams") is None
+    assert self.params.get("DongleId") is not None
+    assert not os.path.isfile(undefined_param)
+
+  def test_params_two_things(self):
+    self.params.put("DongleId", "bob", block=True)
+    self.params.put("AthenadPid", 123, block=True)
+    assert self.params.get("DongleId") == "bob"
+    assert self.params.get("AthenadPid") == 123
+
+  def test_params_get_block(self):
+    def _delayed_writer():
+      time.sleep(0.1)
+      self.params.put("CarParams", b"test", block=True)
+    threading.Thread(target=_delayed_writer).start()
+    assert self.params.get("CarParams") is None
+    assert self.params.get("CarParams", block=True) == b"test"
+
+  def test_params_unknown_key_fails(self):
+    with pytest.raises(UnknownKeyName):
+      self.params.get("swag")
+
+    with pytest.raises(UnknownKeyName):
+      self.params.get_bool("swag")
+
+    with pytest.raises(UnknownKeyName):
+      self.params.put("swag", "abc", block=True)
+
+    with pytest.raises(UnknownKeyName):
+      self.params.put_bool("swag", True, block=True)
+
+  def test_remove_not_there(self):
+    assert self.params.get("CarParams") is None
+    self.params.remove("CarParams")
+    assert self.params.get("CarParams") is None
+
+  def test_get_bool(self):
+    self.params.remove("IsMetric")
+    assert not self.params.get_bool("IsMetric")
+
+    self.params.put_bool("IsMetric", True, block=True)
+    assert self.params.get_bool("IsMetric")
+
+    self.params.put_bool("IsMetric", False, block=True)
+    assert not self.params.get_bool("IsMetric")
+
+    self.params.put("IsMetric", True, block=True)
+    assert self.params.get_bool("IsMetric")
+
+    self.params.put("IsMetric", False, block=True)
+    assert not self.params.get_bool("IsMetric")
+
+  def test_put_non_blocking_with_get_block(self):
+    q = Params()
+    def _delayed_writer():
+      time.sleep(0.1)
+      Params().put("CarParams", b"test")
+    threading.Thread(target=_delayed_writer).start()
+    assert q.get("CarParams") is None
+    assert q.get("CarParams", True) == b"test"
+
+  def test_put_bool_non_blocking_with_get_block(self):
+    q = Params()
+    def _delayed_writer():
+      time.sleep(0.1)
+      Params().put_bool("CarParams", True)
+    threading.Thread(target=_delayed_writer).start()
+    assert q.get("CarParams") is None
+    assert q.get("CarParams", True) == b"1"
+
+  def test_params_all_keys(self):
+    keys = Params().all_keys()
+
+    # sanity checks
+    assert len(keys) > 20
+    assert len(keys) == len(set(keys))
+    assert b"CarParams" in keys
+
+  def test_params_default_value(self):
+    self.params.remove("LanguageSetting")
+    self.params.remove("LongitudinalPersonality")
+    self.params.remove("LiveParameters")
+
+    assert self.params.get("LanguageSetting") is None
+    assert self.params.get("LanguageSetting", return_default=False) is None
+    assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
+    assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
+    assert self.params.get("LiveParameters") is None
+    assert self.params.get("LiveParameters", return_default=True) is None
+
+  def test_params_get_type(self):
+    # json
+    self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
+    assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
+
+    # int
+    self.params.put("BootCount", 1441, block=True)
+    assert self.params.get("BootCount") == 1441
+
+    # bool
+    self.params.put("AdbEnabled", True, block=True)
+    assert self.params.get("AdbEnabled")
+    assert isinstance(self.params.get("AdbEnabled"), bool)
+
+    # time
+    now = datetime.datetime.now(datetime.UTC)
+    self.params.put("InstallDate", now, block=True)
+    assert self.params.get("InstallDate") == now

openpilot/common/tests/test_runner.cc

@@ -0,0 +1,2 @@
+#define CATCH_CONFIG_MAIN
+#include "catch2/catch.hpp"

openpilot/common/tests/test_simple_kalman.py

@@ -0,0 +1,29 @@
+from openpilot.common.simple_kalman import KF1D
+
+
+class TestSimpleKalman:
+  def setup_method(self):
+    dt = 0.01
+    x0_0 = 0.0
+    x1_0 = 0.0
+    A0_0 = 1.0
+    A0_1 = dt
+    A1_0 = 0.0
+    A1_1 = 1.0
+    C0_0 = 1.0
+    C0_1 = 0.0
+    K0_0 = 0.12287673
+    K1_0 = 0.29666309
+
+    self.kf = KF1D(x0=[[x0_0], [x1_0]],
+                   A=[[A0_0, A0_1], [A1_0, A1_1]],
+                   C=[C0_0, C0_1],
+                   K=[[K0_0], [K1_0]])
+
+  def test_getter_setter(self):
+    self.kf.set_x([[1.0], [1.0]])
+    assert self.kf.x == [[1.0], [1.0]]
+
+  def test_update_returns_state(self):
+    x = self.kf.update(100)
+    assert x == [i[0] for i in self.kf.x]

openpilot/common/tests/test_swaglog.cc

@@ -0,0 +1,90 @@
+#include <zmq.h>
+
+#include <iostream>
+
+#include "catch2/catch.hpp"
+#include "common/swaglog.h"
+#include "common/util.h"
+#include "common/version.h"
+#include "common/hardware/hw.h"
+#include "json11/json11.hpp"
+
+#include "sunnypilot/common/version.h"
+
+std::string daemon_name = "testy";
+std::string dongle_id = "test_dongle_id";
+int LINE_NO = 0;
+
+void log_thread(int thread_id, int msg_cnt) {
+  for (int i = 0; i < msg_cnt; ++i) {
+    LOGD("%d", thread_id);
+    LINE_NO = __LINE__ - 1;
+    usleep(1);
+  }
+}
+
+void recv_log(int thread_cnt, int thread_msg_cnt) {
+  void *zctx = zmq_ctx_new();
+  void *sock = zmq_socket(zctx, ZMQ_PULL);
+  zmq_bind(sock, Path::swaglog_ipc().c_str());
+  std::vector<int> thread_msgs(thread_cnt);
+  int total_count = 0;
+
+  for (auto start = std::chrono::steady_clock::now(), now = start;
+       now < start + std::chrono::seconds{1} && total_count < (thread_cnt * thread_msg_cnt);
+       now = std::chrono::steady_clock::now()) {
+    char buf[4096] = {};
+    if (zmq_recv(sock, buf, sizeof(buf), ZMQ_DONTWAIT) <= 0) {
+      if (errno == EAGAIN || errno == EINTR || errno == EFSM) continue;
+      break;
+    }
+
+    REQUIRE(buf[0] == CLOUDLOG_DEBUG);
+    std::string err;
+    auto msg = json11::Json::parse(buf + 1, err);
+    REQUIRE(!msg.is_null());
+
+    REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);
+    REQUIRE_THAT(msg["filename"].string_value(), Catch::Contains("test_swaglog.cc"));
+    REQUIRE(msg["funcname"].string_value() == "log_thread");
+    REQUIRE(msg["lineno"].int_value() == LINE_NO);
+
+    auto ctx = msg["ctx"];
+
+    REQUIRE(ctx["daemon"].string_value() == daemon_name);
+    REQUIRE(ctx["dongle_id"].string_value() == dongle_id);
+    REQUIRE(ctx["dirty"].bool_value() == true);
+
+    REQUIRE(ctx["version"].string_value() == SUNNYPILOT_VERSION);
+
+    std::string device = Hardware::get_name();
+    REQUIRE(ctx["device"].string_value() == device);
+
+    int thread_id = atoi(msg["msg"].string_value().c_str());
+    REQUIRE((thread_id >= 0 && thread_id < thread_cnt));
+    thread_msgs[thread_id]++;
+    total_count++;
+  }
+  for (int i = 0; i < thread_cnt; ++i) {
+    INFO("thread :" << i);
+    REQUIRE(thread_msgs[i] == thread_msg_cnt);
+  }
+  zmq_close(sock);
+  zmq_ctx_destroy(zctx);
+}
+
+TEST_CASE("swaglog") {
+  setenv("MANAGER_DAEMON", daemon_name.c_str(), 1);
+  setenv("DONGLE_ID", dongle_id.c_str(), 1);
+  setenv("dirty", "1", 1);
+  const int thread_cnt = 5;
+  const int thread_msg_cnt = 100;
+
+  std::vector<std::thread> log_threads;
+  for (int i = 0; i < thread_cnt; ++i) {
+    log_threads.push_back(std::thread(log_thread, i, thread_msg_cnt));
+  }
+  for (auto &t : log_threads) t.join();
+
+  recv_log(thread_cnt, thread_msg_cnt);
+}

openpilot/common/tests/test_util.cc

@@ -0,0 +1,147 @@
+
+#include <dirent.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <algorithm>
+#include <climits>
+#include <fstream>
+#include <random>
+#include <string>
+
+#include "catch2/catch.hpp"
+#include "common/util.h"
+
+std::string random_bytes(int size) {
+  std::random_device rd;
+  std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> rbe(rd());
+  std::string bytes(size + 1, '\0');
+  std::generate(bytes.begin(), bytes.end(), std::ref(rbe));
+  return bytes;
+}
+
+TEST_CASE("util::read_file") {
+  SECTION("read /proc/version") {
+    std::string ret = util::read_file("/proc/version");
+    REQUIRE(ret.find("Linux version") != std::string::npos);
+  }
+  SECTION("read from sysfs") {
+    std::string ret = util::read_file("/sys/power/wakeup_count");
+    REQUIRE(!ret.empty());
+  }
+  SECTION("read file") {
+    char filename[] = "/tmp/test_read_XXXXXX";
+    int fd = mkstemp(filename);
+
+    REQUIRE(util::read_file(filename).empty());
+
+    std::string content = random_bytes(64 * 1024);
+    REQUIRE(write(fd, content.c_str(), content.size()) == (ssize_t)content.size());
+    std::string ret = util::read_file(filename);
+    bool equal = (ret == content);
+    REQUIRE(equal);
+    close(fd);
+  }
+  SECTION("read directory") {
+    REQUIRE(util::read_file(".").empty());
+  }
+  SECTION("read non-existent file") {
+    std::string ret = util::read_file("does_not_exist");
+    REQUIRE(ret.empty());
+  }
+  SECTION("read non-permission") {
+    REQUIRE(util::read_file("/proc/kmsg").empty());
+  }
+}
+
+TEST_CASE("util::file_exists") {
+  char filename[] = "/tmp/test_file_exists_XXXXXX";
+  int fd = mkstemp(filename);
+  REQUIRE(fd != -1);
+  close(fd);
+
+  SECTION("existent file") {
+    REQUIRE(util::file_exists(filename));
+    REQUIRE(util::file_exists("/tmp"));
+  }
+  SECTION("nonexistent file") {
+    std::string fn = filename;
+    REQUIRE(!util::file_exists(fn + "/nonexistent"));
+  }
+  SECTION("file has no access permissions") {
+    std::string fn = "/proc/kmsg";
+    std::ifstream f(fn);
+    REQUIRE(f.good() == false);
+    REQUIRE(util::file_exists(fn));
+  }
+  ::remove(filename);
+}
+
+TEST_CASE("util::read_files_in_dir") {
+  char tmp_path[] = "/tmp/test_XXXXXX";
+  const std::string test_path = mkdtemp(tmp_path);
+  const std::string files[] = {".test1", "'test2'", "test3"};
+  for (auto fn : files) {
+    std::ofstream{test_path + "/" + fn} << fn;
+  }
+  mkdir((test_path + "/dir").c_str(), 0777);
+
+  std::map<std::string, std::string> result = util::read_files_in_dir(test_path);
+  REQUIRE(result.find("dir") == result.end());
+  REQUIRE(result.size() == std::size(files));
+  for (auto& [k, v] : result) {
+    REQUIRE(k == v);
+  }
+}
+
+
+TEST_CASE("util::safe_fwrite") {
+  char filename[] = "/tmp/XXXXXX";
+  int fd = mkstemp(filename);
+  close(fd);
+  std::string dat = random_bytes(1024 * 1024);
+
+  FILE *f = util::safe_fopen(filename, "wb");
+  REQUIRE(f != nullptr);
+  size_t size = util::safe_fwrite(dat.data(), 1, dat.size(), f);
+  REQUIRE(size == dat.size());
+  int ret = util::safe_fflush(f);
+  REQUIRE(ret == 0);
+  ret = fclose(f);
+  REQUIRE(ret == 0);
+  bool equal = (dat == util::read_file(filename));
+  REQUIRE(equal);
+}
+
+TEST_CASE("util::create_directories") {
+  REQUIRE(system("rm /tmp/test_create_directories -rf") == 0);
+  std::string dir = "/tmp/test_create_directories/a/b/c/d/e/f";
+
+  auto check_dir_permissions = [](const std::string &path, mode_t mode) -> bool {
+    struct stat st = {};
+    return stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR && (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) == mode;
+  };
+
+  SECTION("create_directories") {
+    REQUIRE(util::create_directories(dir, 0755));
+    REQUIRE(check_dir_permissions(dir, 0755));
+  }
+  SECTION("dir already exists") {
+    REQUIRE(util::create_directories(dir, 0755));
+    REQUIRE(util::create_directories(dir, 0755));
+  }
+  SECTION("a file exists with the same name") {
+    REQUIRE(util::create_directories(dir, 0755));
+    int f = open((dir + "/file").c_str(), O_RDWR | O_CREAT, 0644);
+    REQUIRE(f != -1);
+    close(f);
+    REQUIRE(util::create_directories(dir + "/file", 0755) == false);
+    REQUIRE(util::create_directories(dir + "/file/1/2/3", 0755) == false);
+  }
+  SECTION("end with slashes") {
+    REQUIRE(util::create_directories(dir + "/", 0755));
+  }
+  SECTION("empty") {
+    REQUIRE(util::create_directories("", 0755) == false);
+  }
+}

openpilot/common/text_window.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+import os
+import time
+import subprocess
+from openpilot.common.basedir import BASEDIR
+
+
+class TextWindow:
+  def __init__(self, text):
+    try:
+      self.text_proc = subprocess.Popen(["./text.py", text],
+                                        stdin=subprocess.PIPE,
+                                        cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
+                                        close_fds=True)
+    except OSError:
+      self.text_proc = None
+
+  def get_status(self):
+    if self.text_proc is not None:
+      self.text_proc.poll()
+      return self.text_proc.returncode
+    return None
+
+  def __enter__(self):
+    return self
+
+  def close(self):
+    if self.text_proc is not None:
+      self.text_proc.terminate()
+      self.text_proc = None
+
+  def wait_for_exit(self):
+    if self.text_proc is not None:
+      while True:
+        if self.get_status() == 1:
+          return
+        time.sleep(0.1)
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+if __name__ == "__main__":
+  text = """Traceback (most recent call last):
+  File "./controlsd.py", line 608, in <module>
+    main()
+  File "./controlsd.py", line 604, in main
+    controlsd_thread(sm, pm, logcan)
+  File "./controlsd.py", line 455, in controlsd_thread
+    1/0
+ZeroDivisionError: division by zero"""
+  print(text)
+
+  with TextWindow(text) as s:
+    for _ in range(100):
+      if s.get_status() == 1:
+        print("Got exit button")
+        break
+      time.sleep(0.1)
+  print("gone")

openpilot/common/time_helpers.py

@@ -0,0 +1,16 @@
+import datetime
+from pathlib import Path
+
+MIN_DATE = datetime.datetime(year=2025, month=2, day=21)
+MAX_DATE = datetime.datetime(year=2035, month=1, day=1)
+
+def min_date():
+  # on systemd systems, the default time is the systemd build time
+  systemd_path = Path("/lib/systemd/systemd")
+  if systemd_path.exists():
+    d = datetime.datetime.fromtimestamp(systemd_path.stat().st_mtime)
+    return max(MIN_DATE, d + datetime.timedelta(days=1))
+  return MIN_DATE
+
+def system_time_valid():
+  return min_date() < datetime.datetime.now() < MAX_DATE

openpilot/common/timeout.py

@@ -0,0 +1,27 @@
+import signal
+
+class TimeoutException(Exception):
+  pass
+
+class Timeout:
+  """
+  Timeout context manager.
+  For example this code will raise a TimeoutException:
+  with Timeout(seconds=5, error_msg="Sleep was too long"):
+    time.sleep(10)
+  """
+  def __init__(self, seconds, error_msg=None):
+    if error_msg is None:
+      error_msg = f'Timed out after {seconds} seconds'
+    self.seconds = seconds
+    self.error_msg = error_msg
+
+  def handle_timeout(self, signume, frame):
+    raise TimeoutException(self.error_msg)
+
+  def __enter__(self):
+    signal.signal(signal.SIGALRM, self.handle_timeout)
+    signal.alarm(self.seconds)
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    signal.alarm(0)

openpilot/common/timing.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <cstdint>
+#include <ctime>
+
+#ifdef __APPLE__
+#define CLOCK_BOOTTIME CLOCK_MONOTONIC
+#endif
+
+static inline uint64_t nanos_since_boot() {
+  struct timespec t;
+  clock_gettime(CLOCK_BOOTTIME, &t);
+  return t.tv_sec * 1000000000ULL + t.tv_nsec;
+}
+
+static inline double millis_since_boot() {
+  struct timespec t;
+  clock_gettime(CLOCK_BOOTTIME, &t);
+  return t.tv_sec * 1000.0 + t.tv_nsec * 1e-6;
+}
+
+static inline double seconds_since_boot() {
+  struct timespec t;
+  clock_gettime(CLOCK_BOOTTIME, &t);
+  return (double)t.tv_sec + t.tv_nsec * 1e-9;
+}
+
+static inline uint64_t nanos_since_epoch() {
+  struct timespec t;
+  clock_gettime(CLOCK_REALTIME, &t);
+  return t.tv_sec * 1000000000ULL + t.tv_nsec;
+}
+
+static inline double seconds_since_epoch() {
+  struct timespec t;
+  clock_gettime(CLOCK_REALTIME, &t);
+  return (double)t.tv_sec + t.tv_nsec * 1e-9;
+}
+
+// you probably should use nanos_since_boot instead
+static inline uint64_t nanos_monotonic() {
+  struct timespec t;
+  clock_gettime(CLOCK_MONOTONIC, &t);
+  return t.tv_sec * 1000000000ULL + t.tv_nsec;
+}
+
+static inline uint64_t nanos_monotonic_raw() {
+  struct timespec t;
+  clock_gettime(CLOCK_MONOTONIC_RAW, &t);
+  return t.tv_sec * 1000000000ULL + t.tv_nsec;
+}

openpilot/common/transformations/README.md

@@ -0,0 +1,70 @@
+
+Reference Frames
+------
+Many reference frames are used throughout. This
+folder contains all helper functions needed to
+transform between them. Generally this is done
+by generating a rotation matrix and multiplying.
+
+
+| Name | [x, y, z] | Units | Notes |
+| :-------------: |:-------------:| :-----:| :----: |
+| Geodetic | [Latitude, Longitude, Altitude] | geodetic coordinates | Sometimes used as [lon, lat, alt], avoid this frame. |
+| ECEF | [x, y, z] | meters | We use **ITRF14 (IGS14)**, NOT NAD83. <br> This is the global Mesh3D frame. |
+| NED | [North, East, Down] | meters | Relative to earth's surface, useful for visualizing. |
+| Device | [Forward, Right, Down] | meters | This is the Mesh3D local frame. <br> Relative to camera, **not imu.** <br> ![img](http://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/RPY_angles_of_airplanes.png/440px-RPY_angles_of_airplanes.png)|
+| Calibrated | [Forward, Right, Down] | meters | This is the frame the model outputs are in. <br> More details below. <br>|
+| Car | [Forward, Right, Down] | meters | This is useful for estimating position of points on the road. <br> More details below. <br>|
+| View | [Right, Down, Forward] | meters | Like device frame, but according to camera conventions. |
+| Camera | [u, v, focal] | pixels | Like view frame, but 2d on the camera image.|
+| Normalized Camera | [u / focal, v / focal, 1] | / | |
+| Model | [u, v, focal] | pixels | The sampled rectangle of the full camera frame the model uses. |
+| Normalized Model | [u / focal, v / focal, 1] | / | |
+
+
+
+
+Orientation Conventions
+------
+Quaternions, rotation matrices and euler angles are three
+equivalent representations of orientation and all three are
+used throughout the code base.
+
+For euler angles the preferred convention is [roll, pitch, yaw]
+which corresponds to rotations around the [x, y, z] axes. All
+euler angles should always be in radians or radians/s unless
+for plotting or display purposes. For quaternions the hamilton
+notations is preferred which is [q<sub>w</sub>, q<sub>x</sub>, q<sub>y</sub>, q<sub>z</sub>]. All quaternions
+should always be normalized with a strictly positive q<sub>w</sub>. **These
+quaternions are a unique representation of orientation whereas euler angles
+or rotation matrices are not.**
+
+To rotate from one frame into another with euler angles the
+convention is to rotate around roll, then pitch and then yaw,
+while rotating around the rotated axes, not the original axes.
+
+
+Car frame
+------
+Device frame is aligned with the road-facing camera used by openpilot. However, when controlling the vehicle it is helpful to think in a reference frame aligned with the vehicle. These two reference frames can be different.
+
+The orientation of car frame is defined to be aligned with the car's direction of travel and the road plane when the vehicle is driving on a flat road and not turning. The origin of car frame is defined to be directly below device frame (in car frame), such that it is on the road plane. The position and orientation of this frame is not necessarily always aligned with the direction of travel or the road plane due to suspension movements and other effects.
+
+
+Calibrated frame
+------
+It is helpful for openpilot's driving model to take in images that look similar when mounted differently in different cars. To achieve this we "calibrate" the images by transforming it into calibrated frame. Calibrated frame is defined to be aligned with car frame in pitch and yaw, and aligned with device frame in roll. It also has the same origin as device frame.
+
+
+Example
+------
+To transform global Mesh3D positions and orientations (positions_ecef, quats_ecef) into the local frame described by the
+first position and orientation from Mesh3D one would do:
+```
+ecef_from_local = rot_from_quat(quats_ecef[0])
+local_from_ecef = ecef_from_local.T
+positions_local = np.einsum('ij,kj->ki', local_from_ecef, postions_ecef - positions_ecef[0])
+rotations_global = rot_from_quat(quats_ecef)
+rotations_local = np.einsum('ij,kjl->kil', local_from_ecef, rotations_global)
+eulers_local = euler_from_rot(rotations_local)
+```

openpilot/common/transformations/camera.py

@@ -0,0 +1,179 @@
+import itertools
+import numpy as np
+from dataclasses import dataclass
+
+import openpilot.common.transformations.orientation as orient
+
+## -- hardcoded hardware params --
+@dataclass(frozen=True)
+class CameraConfig:
+  width: int
+  height: int
+  focal_length: float
+
+  @property
+  def size(self):
+    return (self.width, self.height)
+
+  @property
+  def intrinsics(self):
+    # aka 'K' aka camera_frame_from_view_frame
+    return np.array([
+      [self.focal_length,  0.0, float(self.width)/2],
+      [0.0, self.focal_length, float(self.height)/2],
+      [0.0,  0.0, 1.0]
+    ])
+
+  @property
+  def intrinsics_inv(self):
+    # aka 'K_inv' aka view_frame_from_camera_frame
+    return np.linalg.inv(self.intrinsics)
+
+@dataclass(frozen=True)
+class _NoneCameraConfig(CameraConfig):
+  width: int = 0
+  height: int = 0
+  focal_length: float = 0
+
+@dataclass(frozen=True)
+class DeviceCameraConfig:
+  fcam: CameraConfig
+  dcam: CameraConfig
+  ecam: CameraConfig
+
+  def all_cams(self):
+    for cam in ['fcam', 'dcam', 'ecam']:
+      if not isinstance(getattr(self, cam), _NoneCameraConfig):
+        yield cam, getattr(self, cam)
+
+_ar_ox_fisheye = CameraConfig(1928, 1208, 567.0)  # focal length probably wrong? magnification is not consistent across frame
+_os_fisheye = CameraConfig(2688 // 2, 1520 // 2, 567.0 / 4 * 3)
+_ar_ox_config = DeviceCameraConfig(CameraConfig(1928, 1208, 2648.0), _ar_ox_fisheye, _ar_ox_fisheye)
+_os_config = DeviceCameraConfig(CameraConfig(2688 // 2, 1520 // 2, 1522.0 * 3 / 4), _os_fisheye, _os_fisheye)
+_neo_config = DeviceCameraConfig(CameraConfig(1164, 874, 910.0), CameraConfig(816, 612, 650.0), _NoneCameraConfig())
+
+DEVICE_CAMERAS = {
+  # A "device camera" is defined by a device type and sensor
+
+  # sensor type was never set on eon/neo/two
+  ("neo", "unknown"): _neo_config,
+  # unknown here is AR0231, field was added with OX03C10 support
+  ("tici", "unknown"): _ar_ox_config,
+
+  # before deviceState.deviceType was set, assume tici AR config
+  ("unknown", "ar0231"): _ar_ox_config,
+  ("unknown", "ox03c10"): _ar_ox_config,
+
+  # simulator (emulates a tici)
+  ("pc", "unknown"): _ar_ox_config,
+}
+prods = itertools.product(('tici', 'tizi', 'mici'), (('ar0231', _ar_ox_config), ('ox03c10', _ar_ox_config), ('os04c10', _os_config)))
+DEVICE_CAMERAS.update({(d, c[0]): c[1] for d, c in prods})
+
+# device/mesh : x->forward, y-> right, z->down
+# view : x->right, y->down, z->forward
+device_frame_from_view_frame = np.array([
+  [ 0.,  0.,  1.],
+  [ 1.,  0.,  0.],
+  [ 0.,  1.,  0.]
+])
+view_frame_from_device_frame = device_frame_from_view_frame.T
+
+
+# aka 'extrinsic_matrix'
+# road : x->forward, y -> left, z->up
+def get_view_frame_from_road_frame(roll, pitch, yaw, height):
+  device_from_road = orient.rot_from_euler([roll, pitch, yaw]).dot(np.diag([1, -1, -1]))
+  view_from_road = view_frame_from_device_frame.dot(device_from_road)
+  return np.hstack((view_from_road, [[0], [height], [0]]))
+
+
+
+# aka 'extrinsic_matrix'
+def get_view_frame_from_calib_frame(roll, pitch, yaw, height):
+  device_from_calib= orient.rot_from_euler([roll, pitch, yaw])
+  view_from_calib = view_frame_from_device_frame.dot(device_from_calib)
+  return np.hstack((view_from_calib, [[0], [height], [0]]))
+
+
+def vp_from_ke(m):
+  """
+  Computes the vanishing point from the product of the intrinsic and extrinsic
+  matrices C = KE.
+
+  The vanishing point is defined as lim x->infinity C (x, 0, 0, 1).T
+  """
+  return (m[0, 0]/m[2, 0], m[1, 0]/m[2, 0])
+
+
+def roll_from_ke(m):
+  # note: different from calibration.h/RollAnglefromKE: i think that one's just wrong
+  return np.arctan2(-(m[1, 0] - m[1, 1] * m[2, 0] / m[2, 1]),
+                    -(m[0, 0] - m[0, 1] * m[2, 0] / m[2, 1]))
+
+
+def normalize(img_pts, intrinsics):
+  # normalizes image coordinates
+  # accepts single pt or array of pts
+  intrinsics_inv = np.linalg.inv(intrinsics)
+  img_pts = np.array(img_pts)
+  input_shape = img_pts.shape
+  img_pts = np.atleast_2d(img_pts)
+  img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
+  img_pts_normalized = img_pts.dot(intrinsics_inv.T)
+  img_pts_normalized[(img_pts < 0).any(axis=1)] = np.nan
+  return img_pts_normalized[:, :2].reshape(input_shape)
+
+
+def denormalize(img_pts, intrinsics, width=np.inf, height=np.inf):
+  # denormalizes image coordinates
+  # accepts single pt or array of pts
+  img_pts = np.array(img_pts)
+  input_shape = img_pts.shape
+  img_pts = np.atleast_2d(img_pts)
+  img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1), dtype=img_pts.dtype)))
+  img_pts_denormalized = img_pts.dot(intrinsics.T)
+  if np.isfinite(width):
+    img_pts_denormalized[img_pts_denormalized[:, 0] > width] = np.nan
+    img_pts_denormalized[img_pts_denormalized[:, 0] < 0] = np.nan
+  if np.isfinite(height):
+    img_pts_denormalized[img_pts_denormalized[:, 1] > height] = np.nan
+    img_pts_denormalized[img_pts_denormalized[:, 1] < 0] = np.nan
+  return img_pts_denormalized[:, :2].reshape(input_shape)
+
+
+def get_calib_from_vp(vp, intrinsics):
+  vp_norm = normalize(vp, intrinsics)
+  yaw_calib = np.arctan(vp_norm[0])
+  pitch_calib = -np.arctan(vp_norm[1]*np.cos(yaw_calib))
+  roll_calib = 0
+  return roll_calib, pitch_calib, yaw_calib
+
+
+def device_from_ecef(pos_ecef, orientation_ecef, pt_ecef):
+  # device from ecef frame
+  # device frame is x -> forward, y-> right, z -> down
+  # accepts single pt or array of pts
+  input_shape = pt_ecef.shape
+  pt_ecef = np.atleast_2d(pt_ecef)
+  ecef_from_device_rot = orient.rotations_from_quats(orientation_ecef)
+  device_from_ecef_rot = ecef_from_device_rot.T
+  pt_ecef_rel = pt_ecef - pos_ecef
+  pt_device = np.einsum('jk,ik->ij', device_from_ecef_rot, pt_ecef_rel)
+  return pt_device.reshape(input_shape)
+
+
+def img_from_device(pt_device):
+  # img coordinates from pts in device frame
+  # first transforms to view frame, then to img coords
+  # accepts single pt or array of pts
+  input_shape = pt_device.shape
+  pt_device = np.atleast_2d(pt_device)
+  pt_view = np.einsum('jk,ik->ij', view_frame_from_device_frame, pt_device)
+
+  # This function should never return negative depths
+  pt_view[pt_view[:, 2] < 0] = np.nan
+
+  pt_img = pt_view/pt_view[:, 2:3]
+  return pt_img.reshape(input_shape)[:, :2]
+

openpilot/common/transformations/coordinates.py

@@ -0,0 +1,18 @@
+from openpilot.common.transformations.orientation import numpy_wrap
+from openpilot.common.transformations.transformations import (ecef2geodetic_single,
+                                                    geodetic2ecef_single)
+from openpilot.common.transformations.transformations import LocalCoord as LocalCoord_single
+
+
+class LocalCoord(LocalCoord_single):
+  ecef2ned = numpy_wrap(LocalCoord_single.ecef2ned_single, (3,), (3,))
+  ned2ecef = numpy_wrap(LocalCoord_single.ned2ecef_single, (3,), (3,))
+  geodetic2ned = numpy_wrap(LocalCoord_single.geodetic2ned_single, (3,), (3,))
+  ned2geodetic = numpy_wrap(LocalCoord_single.ned2geodetic_single, (3,), (3,))
+
+
+geodetic2ecef = numpy_wrap(geodetic2ecef_single, (3,), (3,))
+ecef2geodetic = numpy_wrap(ecef2geodetic_single, (3,), (3,))
+
+geodetic_from_ecef = ecef2geodetic
+ecef_from_geodetic = geodetic2ecef

openpilot/common/transformations/model.py

@@ -0,0 +1,70 @@
+import numpy as np
+
+from openpilot.common.transformations.orientation import rot_from_euler
+from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame, _ar_ox_fisheye
+
+# segnet
+SEGNET_SIZE = (512, 384)
+
+# MED model
+MEDMODEL_INPUT_SIZE = (512, 256)
+MEDMODEL_YUV_SIZE = (MEDMODEL_INPUT_SIZE[0], MEDMODEL_INPUT_SIZE[1] * 3 // 2)
+MEDMODEL_CY = 47.6
+
+medmodel_fl = 910.0
+medmodel_intrinsics = np.array([
+  [medmodel_fl,  0.0,  0.5 * MEDMODEL_INPUT_SIZE[0]],
+  [0.0,  medmodel_fl,                   MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
+
+# BIG model
+BIGMODEL_INPUT_SIZE = (1024, 512)
+BIGMODEL_YUV_SIZE = (BIGMODEL_INPUT_SIZE[0], BIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+bigmodel_fl = 910.0
+bigmodel_intrinsics = np.array([
+  [bigmodel_fl,  0.0,  0.5 * BIGMODEL_INPUT_SIZE[0]],
+  [0.0,  bigmodel_fl,             256 + MEDMODEL_CY],
+  [0.0,  0.0,                                   1.0]])
+
+
+# SBIG model (big model with the size of small model)
+SBIGMODEL_INPUT_SIZE = (512, 256)
+SBIGMODEL_YUV_SIZE = (SBIGMODEL_INPUT_SIZE[0], SBIGMODEL_INPUT_SIZE[1] * 3 // 2)
+
+sbigmodel_fl = 455.0
+sbigmodel_intrinsics = np.array([
+  [sbigmodel_fl,  0.0,  0.5 * SBIGMODEL_INPUT_SIZE[0]],
+  [0.0,  sbigmodel_fl,      0.5 * (256 + MEDMODEL_CY)],
+  [0.0,  0.0,                                     1.0]])
+
+DM_INPUT_SIZE = (1440, 960)
+dmonitoringmodel_fl = _ar_ox_fisheye.focal_length
+dmonitoringmodel_intrinsics = np.array([
+  [dmonitoringmodel_fl,  0.0, DM_INPUT_SIZE[0]/2],
+  [0.0, dmonitoringmodel_fl, DM_INPUT_SIZE[1]/2 - (_ar_ox_fisheye.height - DM_INPUT_SIZE[1])/2],
+  [0.0,  0.0, 1.0]])
+
+bigmodel_frame_from_calib_frame = np.dot(bigmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+
+sbigmodel_frame_from_calib_frame = np.dot(sbigmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+medmodel_frame_from_calib_frame = np.dot(medmodel_intrinsics,
+  get_view_frame_from_calib_frame(0, 0, 0, 0))
+
+medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(bigmodel_intrinsics))
+
+calib_from_medmodel = np.linalg.inv(medmodel_frame_from_calib_frame[:, :3])
+calib_from_sbigmodel = np.linalg.inv(sbigmodel_frame_from_calib_frame[:, :3])
+
+# This function is verified to give similar results to xx.uncommon.utils.transform_img
+def get_warp_matrix(device_from_calib_euler: np.ndarray, intrinsics: np.ndarray, bigmodel_frame: bool = False) -> np.ndarray:
+  calib_from_model = calib_from_sbigmodel if bigmodel_frame else calib_from_medmodel
+  device_from_calib = rot_from_euler(device_from_calib_euler)
+  camera_from_calib = intrinsics @ view_frame_from_device_frame @ device_from_calib
+  warp_matrix: np.ndarray = camera_from_calib @ calib_from_model
+  return warp_matrix

openpilot/common/transformations/orientation.py

@@ -0,0 +1,51 @@
+import numpy as np
+from collections.abc import Callable
+
+from openpilot.common.transformations.transformations import (ecef_euler_from_ned_single,
+                                                    euler2quat_single,
+                                                    euler2rot_single,
+                                                    ned_euler_from_ecef_single,
+                                                    quat2euler_single,
+                                                    quat2rot_single,
+                                                    rot2euler_single,
+                                                    rot2quat_single)
+
+
+def numpy_wrap(function, input_shape, output_shape) -> Callable[..., np.ndarray]:
+  """Wrap a function to take either an input or list of inputs and return the correct shape"""
+  def f(*inps):
+    *args, inp = inps
+    inp = np.array(inp)
+    shape = inp.shape
+
+    if len(shape) == len(input_shape):
+      out_shape = output_shape
+    else:
+      out_shape = (shape[0],) + output_shape
+
+    # Add empty dimension if inputs is not a list
+    if len(shape) == len(input_shape):
+      inp = inp.reshape((1, ) + inp.shape)
+
+    result = np.asarray([function(*args, i) for i in inp])
+    return result.reshape(out_shape)
+  return f
+
+
+euler2quat = numpy_wrap(euler2quat_single, (3,), (4,))
+quat2euler = numpy_wrap(quat2euler_single, (4,), (3,))
+quat2rot = numpy_wrap(quat2rot_single, (4,), (3, 3))
+rot2quat = numpy_wrap(rot2quat_single, (3, 3), (4,))
+euler2rot = numpy_wrap(euler2rot_single, (3,), (3, 3))
+rot2euler = numpy_wrap(rot2euler_single, (3, 3), (3,))
+ecef_euler_from_ned = numpy_wrap(ecef_euler_from_ned_single, (3,), (3,))
+ned_euler_from_ecef = numpy_wrap(ned_euler_from_ecef_single, (3,), (3,))
+
+quats_from_rotations = rot2quat
+quat_from_rot = rot2quat
+rotations_from_quats = quat2rot
+rot_from_quat = quat2rot
+euler_from_rot = rot2euler
+euler_from_quat = quat2euler
+rot_from_euler = euler2rot
+quat_from_euler = euler2quat

openpilot/common/transformations/tests/test_coordinates.py

@@ -0,0 +1,137 @@
+import numpy as np
+
+import openpilot.common.transformations.coordinates as coord
+
+geodetic_positions = np.array([[37.7610403, -122.4778699, 115],
+                                 [27.4840915, -68.5867592, 2380],
+                                 [32.4916858, -113.652821, -6],
+                                 [15.1392514, 103.6976037, 24],
+                                 [24.2302229, 44.2835412, 1650]])
+
+ecef_positions = np.array([[-2711076.55270557, -4259167.14692758,  3884579.87669935],
+                          [ 2068042.69652729, -5273435.40316622,  2927004.89190746],
+                          [-2160412.60461669, -4932588.89873832,  3406542.29652851],
+                          [-1458247.92550567,  5983060.87496612,  1654984.6099885 ],
+                          [ 4167239.10867871,  4064301.90363223,  2602234.6065749 ]])
+
+ecef_positions_offset = np.array([[-2711004.46961115, -4259099.33540613,  3884605.16002147],
+                                  [ 2068074.30639499, -5273413.78835412,  2927012.48741131],
+                                  [-2160344.53748176, -4932586.20092211,  3406636.2962545 ],
+                                  [-1458211.98517094,  5983151.11161276,  1655077.02698447],
+                                  [ 4167271.20055269,  4064398.22619263,  2602238.95265847]])
+
+
+ned_offsets = np.array([[78.722153649976391, 24.396208657446344, 60.343017506838436],
+                       [10.699003365155221, 37.319278617604269, 4.1084100025050407],
+                       [95.282646251726959, 61.266689955574428, -25.376506058505054],
+                       [68.535769283630003, -56.285970011848889, -100.54840137956515],
+                       [-33.066609321880179, 46.549821994306861, -84.062540548335591]])
+
+ecef_init_batch = np.array([2068042.69652729, -5273435.40316622,  2927004.89190746])
+ecef_positions_offset_batch = np.array([[ 2068089.41454771, -5273434.46829148,  2927074.04783672],
+                                        [ 2068103.31628647, -5273393.92275431,  2927102.08725987],
+                                        [ 2068108.49939636, -5273359.27047121,  2927045.07091581],
+                                        [ 2068075.12395611, -5273381.69432566,  2927041.08207992],
+                                        [ 2068060.72033399, -5273430.6061505,  2927094.54928305]])
+
+ned_offsets_batch = np.array([[  53.88103168,   43.83445935,  -46.27488057],
+                              [  93.83378995,   71.57943024,  -30.23113187],
+                              [  57.26725796,   89.05602684,   23.02265814],
+                              [  49.71775195,   49.79767572,   17.15351015],
+                              [  78.56272609,   18.53100158,  -43.25290759]])
+
+
+class TestNED:
+  def test_small_distances(self):
+    start_geodetic = np.array([33.8042184, -117.888593, 0.0])
+    local_coord = coord.LocalCoord.from_geodetic(start_geodetic)
+
+    start_ned = local_coord.geodetic2ned(start_geodetic)
+    np.testing.assert_array_equal(start_ned, np.zeros(3,))
+
+    west_geodetic = start_geodetic + [0, -0.0005, 0]
+    west_ned = local_coord.geodetic2ned(west_geodetic)
+    assert np.abs(west_ned[0]) < 1e-3
+    assert west_ned[1] < 0
+
+    southwest_geodetic = start_geodetic + [-0.0005, -0.002, 0]
+    southwest_ned = local_coord.geodetic2ned(southwest_geodetic)
+    assert southwest_ned[0] < 0
+    assert southwest_ned[1] < 0
+
+  def test_ecef_geodetic(self):
+    # testing single
+    np.testing.assert_allclose(ecef_positions[0], coord.geodetic2ecef(geodetic_positions[0]), rtol=1e-9)
+    np.testing.assert_allclose(geodetic_positions[0, :2], coord.ecef2geodetic(ecef_positions[0])[:2], rtol=1e-9)
+    np.testing.assert_allclose(geodetic_positions[0, 2], coord.ecef2geodetic(ecef_positions[0])[2], rtol=1e-9, atol=1e-4)
+
+    np.testing.assert_allclose(geodetic_positions[:, :2], coord.ecef2geodetic(ecef_positions)[:, :2], rtol=1e-9)
+    np.testing.assert_allclose(geodetic_positions[:, 2], coord.ecef2geodetic(ecef_positions)[:, 2], rtol=1e-9, atol=1e-4)
+    np.testing.assert_allclose(ecef_positions, coord.geodetic2ecef(geodetic_positions), rtol=1e-9)
+
+
+  def test_ned(self):
+    for ecef_pos in ecef_positions:
+      converter = coord.LocalCoord.from_ecef(ecef_pos)
+      ecef_pos_moved = ecef_pos + [25, -25, 25]
+      ecef_pos_moved_double_converted = converter.ned2ecef(converter.ecef2ned(ecef_pos_moved))
+      np.testing.assert_allclose(ecef_pos_moved, ecef_pos_moved_double_converted, rtol=1e-9)
+
+    for geo_pos in geodetic_positions:
+      converter = coord.LocalCoord.from_geodetic(geo_pos)
+      geo_pos_moved = geo_pos + np.array([0, 0, 10])
+      geo_pos_double_converted_moved = converter.ned2geodetic(converter.geodetic2ned(geo_pos) + np.array([0, 0, -10]))
+      np.testing.assert_allclose(geo_pos_moved[:2], geo_pos_double_converted_moved[:2], rtol=1e-9, atol=1e-6)
+      np.testing.assert_allclose(geo_pos_moved[2], geo_pos_double_converted_moved[2], rtol=1e-9, atol=1e-4)
+
+  def test_ned_saved_results(self):
+    for i, ecef_pos in enumerate(ecef_positions):
+      converter = coord.LocalCoord.from_ecef(ecef_pos)
+      np.testing.assert_allclose(converter.ned2ecef(ned_offsets[i]),
+                                 ecef_positions_offset[i],
+                                 rtol=1e-9, atol=1e-4)
+      np.testing.assert_allclose(converter.ecef2ned(ecef_positions_offset[i]),
+                                 ned_offsets[i],
+                                 rtol=1e-9, atol=1e-4)
+
+  def test_ned_batch(self):
+    converter = coord.LocalCoord.from_ecef(ecef_init_batch)
+    np.testing.assert_allclose(converter.ecef2ned(ecef_positions_offset_batch),
+                                                           ned_offsets_batch,
+                                                           rtol=1e-9, atol=1e-7)
+    np.testing.assert_allclose(converter.ned2ecef(ned_offsets_batch),
+                                                           ecef_positions_offset_batch,
+                                                           rtol=1e-9, atol=1e-7)
+
+  def test_errors(self):
+    # Test wrong shape/type for geodetic2ecef
+    # numpy_wrap raises IndexError for scalar input
+    with np.testing.assert_raises(IndexError):
+      coord.geodetic2ecef(1.0)
+
+    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
+      coord.geodetic2ecef([0, 0])
+
+    with np.testing.assert_raises_regex(ValueError, "Geodetic must be size 3"):
+      coord.geodetic2ecef([0, 0, 0, 0])
+
+    with np.testing.assert_raises(TypeError):
+      coord.geodetic2ecef(['a', 'b', 'c'])
+
+    # Test LocalCoord constructor errors
+    with np.testing.assert_raises(ValueError):
+      coord.LocalCoord.from_geodetic([0, 0])
+
+    with np.testing.assert_raises(ValueError):
+      coord.LocalCoord.from_geodetic(1)
+
+    with np.testing.assert_raises(TypeError):
+      coord.LocalCoord.from_geodetic(['a', 'b', 'c'])
+
+    # Test wrong shape/type for ecef2geodetic
+    with np.testing.assert_raises(ValueError):
+      coord.ecef2geodetic([1, 2])
+    with np.testing.assert_raises(ValueError):
+      coord.ecef2geodetic([1, 2, 3, 4])
+    with np.testing.assert_raises(IndexError):
+      coord.ecef2geodetic(1.0)

openpilot/common/transformations/tests/test_orientation.py

@@ -0,0 +1,91 @@
+import numpy as np
+import pytest
+
+from openpilot.common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
+                                               rot2quat, quat2rot, \
+                                               ned_euler_from_ecef
+
+eulers = np.array([[ 1.46520501,  2.78688383,  2.92780854],
+       [ 4.86909526,  3.60618161,  4.30648981],
+       [ 3.72175965,  2.68763705,  5.43895988],
+       [ 5.92306687,  5.69573614,  0.81100357],
+       [ 0.67838374,  5.02402037,  2.47106426]])
+
+quats = np.array([[ 0.66855182, -0.71500939,  0.19539353,  0.06017818],
+       [ 0.43163717,  0.70013301,  0.28209145,  0.49389021],
+       [ 0.44121991, -0.08252646,  0.34257534,  0.82532207],
+       [ 0.88578382, -0.04515356, -0.32936046,  0.32383617],
+       [ 0.06578165,  0.61282835,  0.07126891,  0.78424163]])
+
+ecef_positions = np.array([[-2711076.55270557, -4259167.14692758,  3884579.87669935],
+                          [ 2068042.69652729, -5273435.40316622,  2927004.89190746],
+                          [-2160412.60461669, -4932588.89873832,  3406542.29652851],
+                          [-1458247.92550567,  5983060.87496612,  1654984.6099885 ],
+                          [ 4167239.10867871,  4064301.90363223,  2602234.6065749 ]])
+
+ned_eulers = np.array([[ 0.46806039, -0.4881889 ,  1.65697808],
+                    [-2.14525969, -0.36533066,  0.73813479],
+                    [-1.39523364, -0.58540761, -1.77376356],
+                    [-1.84220435,  0.61828016, -1.03310421],
+                    [ 2.50450101,  0.36304151,  0.33136365]])
+
+
+class TestOrientation:
+  def test_quat_euler(self):
+    for i, eul in enumerate(eulers):
+      np.testing.assert_allclose(quats[i], euler2quat(eul), rtol=1e-7)
+      np.testing.assert_allclose(quats[i], euler2quat(quat2euler(quats[i])), rtol=1e-6)
+    for i, eul in enumerate(eulers):
+      np.testing.assert_allclose(quats[i], euler2quat(list(eul)), rtol=1e-7)
+      np.testing.assert_allclose(quats[i], euler2quat(quat2euler(list(quats[i]))), rtol=1e-6)
+    np.testing.assert_allclose(quats, euler2quat(eulers), rtol=1e-7)
+    np.testing.assert_allclose(quats, euler2quat(quat2euler(quats)), rtol=1e-6)
+
+  def test_rot_euler(self):
+    for eul in eulers:
+      np.testing.assert_allclose(euler2quat(eul), euler2quat(rot2euler(euler2rot(eul))), rtol=1e-7)
+    for eul in eulers:
+      np.testing.assert_allclose(euler2quat(eul), euler2quat(rot2euler(euler2rot(list(eul)))), rtol=1e-7)
+    np.testing.assert_allclose(euler2quat(eulers), euler2quat(rot2euler(euler2rot(eulers))), rtol=1e-7)
+
+  def test_rot_quat(self):
+    for quat in quats:
+      np.testing.assert_allclose(quat, rot2quat(quat2rot(quat)), rtol=1e-7)
+    for quat in quats:
+      np.testing.assert_allclose(quat, rot2quat(quat2rot(list(quat))), rtol=1e-7)
+    np.testing.assert_allclose(quats, rot2quat(quat2rot(quats)), rtol=1e-7)
+
+  def test_euler_ned(self):
+    for i in range(len(eulers)):
+      np.testing.assert_allclose(ned_eulers[i], ned_euler_from_ecef(ecef_positions[i], eulers[i]), rtol=1e-7)
+      #np.testing.assert_allclose(eulers[i], ecef_euler_from_ned(ecef_positions[i], ned_eulers[i]), rtol=1e-7)
+    # np.testing.assert_allclose(ned_eulers, ned_euler_from_ecef(ecef_positions, eulers), rtol=1e-7)
+
+  def test_inputs(self):
+    with pytest.raises(ValueError):
+      euler2quat([1, 2])
+
+    with pytest.raises(ValueError):
+      quat2rot([1, 2, 3])
+
+    with pytest.raises(IndexError):
+      rot2quat(np.zeros((2, 2)))
+
+  def test_euler_rot_consistency(self):
+    rpy = [0.1, 0.2, 0.3]
+    R = euler2rot(rpy)
+
+    # R -> q -> R
+    q = rot2quat(R)
+    R_new = quat2rot(q)
+    np.testing.assert_allclose(R, R_new, atol=1e-15)
+
+    # q -> R -> Euler (quat2euler) -> R
+    rpy_new = quat2euler(q)
+    R_new2 = euler2rot(rpy_new)
+    np.testing.assert_allclose(R, R_new2, atol=1e-15)
+
+    # R -> Euler (rot2euler) -> R
+    rpy_from_rot = rot2euler(R)
+    R_new3 = euler2rot(rpy_from_rot)
+    np.testing.assert_allclose(R, R_new3, atol=1e-15)

openpilot/common/transformations/transformations.py

@@ -0,0 +1,342 @@
+import numpy as np
+
+
+# Constants
+a = 6378137.0
+b = 6356752.3142
+esq = 6.69437999014e-3
+e1sq = 6.73949674228e-3
+
+
+def geodetic2ecef_single(g):
+  """
+  Convert geodetic coordinates (latitude, longitude, altitude) to ECEF.
+  """
+  try:
+    if len(g) != 3:
+      raise ValueError("Geodetic must be size 3")
+  except TypeError:
+    raise ValueError("Geodetic must be a sequence of length 3") from None
+
+  lat, lon, alt = g
+  lat = np.radians(lat)
+  lon = np.radians(lon)
+  xi = np.sqrt(1.0 - esq * np.sin(lat)**2)
+  x = (a / xi + alt) * np.cos(lat) * np.cos(lon)
+  y = (a / xi + alt) * np.cos(lat) * np.sin(lon)
+  z = (a / xi * (1.0 - esq) + alt) * np.sin(lat)
+  return np.array([x, y, z])
+
+
+def ecef2geodetic_single(e):
+  """
+  Convert ECEF to geodetic coordinates using Ferrari's solution.
+  """
+  x, y, z = e
+  r = np.sqrt(x**2 + y**2)
+  Esq = a**2 - b**2
+  F = 54 * b**2 * z**2
+  G = r**2 + (1 - esq) * z**2 - esq * Esq
+  C = (esq**2 * F * r**2) / (G**3)
+  S = np.cbrt(1 + C + np.sqrt(C**2 + 2 * C))
+  P = F / (3 * (S + 1 / S + 1)**2 * G**2)
+  Q = np.sqrt(1 + 2 * esq**2 * P)
+  r_0 = -(P * esq * r) / (1 + Q) + np.sqrt(0.5 * a**2 * (1 + 1.0 / Q) - P * (1 - esq) * z**2 / (Q * (1 + Q)) - 0.5 * P * r**2)
+  U = np.sqrt((r - esq * r_0)**2 + z**2)
+  V = np.sqrt((r - esq * r_0)**2 + (1 - esq) * z**2)
+  Z_0 = b**2 * z / (a * V)
+  h = U * (1 - b**2 / (a * V))
+  lat = np.arctan((z + e1sq * Z_0) / r)
+  lon = np.arctan2(y, x)
+  return np.array([np.degrees(lat), np.degrees(lon), h])
+
+
+def euler2quat_single(euler):
+  """
+  Convert Euler angles (roll, pitch, yaw) to a quaternion.
+  Rotation order: Z-Y-X (yaw, pitch, roll).
+  """
+  phi, theta, psi = euler
+
+  c_phi, s_phi = np.cos(phi / 2), np.sin(phi / 2)
+  c_theta, s_theta = np.cos(theta / 2), np.sin(theta / 2)
+  c_psi, s_psi = np.cos(psi / 2), np.sin(psi / 2)
+
+  w = c_phi * c_theta * c_psi + s_phi * s_theta * s_psi
+  x = s_phi * c_theta * c_psi - c_phi * s_theta * s_psi
+  y = c_phi * s_theta * c_psi + s_phi * c_theta * s_psi
+  z = c_phi * c_theta * s_psi - s_phi * s_theta * c_psi
+
+  if w < 0:
+    return np.array([-w, -x, -y, -z])
+  return np.array([w, x, y, z])
+
+
+def quat2euler_single(q):
+  """
+  Convert a quaternion to Euler angles (roll, pitch, yaw).
+  """
+  w, x, y, z = q
+  gamma = np.arctan2(2 * (w * x + y * z), 1 - 2 * (x**2 + y**2))
+  sin_arg = 2 * (w * y - z * x)
+  sin_arg = np.clip(sin_arg, -1.0, 1.0)
+  theta = np.arcsin(sin_arg)
+  psi = np.arctan2(2 * (w * z + x * y), 1 - 2 * (y**2 + z**2))
+  return np.array([gamma, theta, psi])
+
+
+def quat2rot_single(q):
+  """
+  Convert a quaternion to a 3x3 rotation matrix.
+  """
+  w, x, y, z = q
+  xx, yy, zz = x * x, y * y, z * z
+  xy, xz, yz = x * y, x * z, y * z
+  wx, wy, wz = w * x, w * y, w * z
+
+  mat = np.array([
+    [1 - 2 * (yy + zz), 2 * (xy - wz), 2 * (xz + wy)],
+    [2 * (xy + wz), 1 - 2 * (xx + zz), 2 * (yz - wx)],
+    [2 * (xz - wy), 2 * (yz + wx), 1 - 2 * (xx + yy)]
+  ])
+  return mat
+
+
+def rot2quat_single(rot):
+  """
+  Convert a 3x3 rotation matrix to a quaternion.
+  """
+  trace = np.trace(rot)
+  if trace > 0:
+    s = 0.5 / np.sqrt(trace + 1.0)
+    w = 0.25 / s
+    x = (rot[2, 1] - rot[1, 2]) * s
+    y = (rot[0, 2] - rot[2, 0]) * s
+    z = (rot[1, 0] - rot[0, 1]) * s
+  else:
+    if rot[0, 0] > rot[1, 1] and rot[0, 0] > rot[2, 2]:
+      s = 2.0 * np.sqrt(1.0 + rot[0, 0] - rot[1, 1] - rot[2, 2])
+      w = (rot[2, 1] - rot[1, 2]) / s
+      x = 0.25 * s
+      y = (rot[0, 1] + rot[1, 0]) / s
+      z = (rot[0, 2] + rot[2, 0]) / s
+    elif rot[1, 1] > rot[2, 2]:
+      s = 2.0 * np.sqrt(1.0 + rot[1, 1] - rot[0, 0] - rot[2, 2])
+      w = (rot[0, 2] - rot[2, 0]) / s
+      x = (rot[0, 1] + rot[1, 0]) / s
+      y = 0.25 * s
+      z = (rot[1, 2] + rot[2, 1]) / s
+    else:
+      s = 2.0 * np.sqrt(1.0 + rot[2, 2] - rot[0, 0] - rot[1, 1])
+      w = (rot[1, 0] - rot[0, 1]) / s
+      x = (rot[0, 2] + rot[2, 0]) / s
+      y = (rot[1, 2] + rot[2, 1]) / s
+      z = 0.25 * s
+
+  if w < 0:
+    return np.array([-w, -x, -y, -z])
+  return np.array([w, x, y, z])
+
+
+def euler2rot_single(euler):
+  """
+  Convert Euler angles (roll, pitch, yaw) to a 3x3 rotation matrix.
+  Rotation order: Z-Y-X (yaw, pitch, roll).
+  """
+  phi, theta, psi = euler
+
+  cx, sx = np.cos(phi), np.sin(phi)
+  cy, sy = np.cos(theta), np.sin(theta)
+  cz, sz = np.cos(psi), np.sin(psi)
+
+  Rx = np.array([[1, 0, 0], [0, cx, -sx], [0, sx, cx]])
+  Ry = np.array([[cy, 0, sy], [0, 1, 0], [-sy, 0, cy]])
+  Rz = np.array([[cz, -sz, 0], [sz, cz, 0], [0, 0, 1]])
+
+  return Rz @ Ry @ Rx
+
+
+def rot2euler_single(rot):
+  """
+  Convert a 3x3 rotation matrix to Euler angles (roll, pitch, yaw).
+  """
+  return quat2euler_single(rot2quat_single(rot))
+
+
+def rot_matrix(roll, pitch, yaw):
+  """
+  Create a 3x3 rotation matrix from roll, pitch, and yaw angles.
+  """
+  return euler2rot_single([roll, pitch, yaw])
+
+
+def axis_angle_to_rot(axis, angle):
+  """
+  Convert an axis-angle representation to a 3x3 rotation matrix.
+  """
+  c = np.cos(angle / 2)
+  s = np.sin(angle / 2)
+  q = np.array([c, s*axis[0], s*axis[1], s*axis[2]])
+  return quat2rot_single(q)
+
+
+class LocalCoord:
+  """
+  A class to handle conversions between ECEF and local NED coordinates.
+  """
+  def __init__(self, geodetic=None, ecef=None):
+    """
+    Initialize LocalCoord with either geodetic or ECEF coordinates.
+    """
+    if geodetic is not None:
+      self.init_ecef = geodetic2ecef_single(geodetic)
+      lat, lon, _ = geodetic
+    elif ecef is not None:
+      self.init_ecef = np.array(ecef)
+      lat, lon, _ = ecef2geodetic_single(ecef)
+    else:
+      raise ValueError("Must provide geodetic or ecef")
+
+    lat = np.radians(lat)
+    lon = np.radians(lon)
+
+    self.ned2ecef_matrix = np.array([
+      [-np.sin(lat) * np.cos(lon), -np.sin(lon), -np.cos(lat) * np.cos(lon)],
+      [-np.sin(lat) * np.sin(lon), np.cos(lon), -np.cos(lat) * np.sin(lon)],
+      [np.cos(lat), 0, -np.sin(lat)]
+    ])
+    self.ecef2ned_matrix = self.ned2ecef_matrix.T
+
+  @classmethod
+  def from_geodetic(cls, geodetic):
+    """
+    Create a LocalCoord instance from geodetic coordinates.
+    """
+    return cls(geodetic=geodetic)
+
+  @classmethod
+  def from_ecef(cls, ecef):
+    """
+    Create a LocalCoord instance from ECEF coordinates.
+    """
+    return cls(ecef=ecef)
+
+  def ecef2ned_single(self, ecef):
+    """
+    Convert a single ECEF point to NED coordinates relative to the origin.
+    """
+    return self.ecef2ned_matrix @ (ecef - self.init_ecef)
+
+  def ned2ecef_single(self, ned):
+    """
+    Convert a single NED point to ECEF coordinates.
+    """
+    return self.ned2ecef_matrix @ ned + self.init_ecef
+
+  def geodetic2ned_single(self, geodetic):
+    """
+    Convert a single geodetic point to NED coordinates.
+    """
+    ecef = geodetic2ecef_single(geodetic)
+    return self.ecef2ned_single(ecef)
+
+  def ned2geodetic_single(self, ned):
+    """
+    Convert a single NED point to geodetic coordinates.
+    """
+    ecef = self.ned2ecef_single(ned)
+    return ecef2geodetic_single(ecef)
+
+  @property
+  def ned_from_ecef_matrix(self):
+    """
+    Returns the rotation matrix from ECEF to NED coordinates.
+    """
+    return self.ecef2ned_matrix
+
+  @property
+  def ecef_from_ned_matrix(self):
+    """
+    Returns the rotation matrix from NED to ECEF coordinates.
+    """
+    return self.ned2ecef_matrix
+
+
+def ecef_euler_from_ned_single(ecef_init, ned_pose):
+  """
+  Convert NED Euler angles (roll, pitch, yaw) at a given ECEF origin
+  to equivalent ECEF Euler angles.
+  """
+  converter = LocalCoord(ecef=ecef_init)
+  zero = np.array(ecef_init)
+
+  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
+  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
+  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
+
+  phi, theta, psi = ned_pose
+
+  x1 = axis_angle_to_rot(z0, psi) @ x0
+  y1 = axis_angle_to_rot(z0, psi) @ y0
+  z1 = axis_angle_to_rot(z0, psi) @ z0
+
+  x2 = axis_angle_to_rot(y1, theta) @ x1
+  y2 = axis_angle_to_rot(y1, theta) @ y1
+  z2 = axis_angle_to_rot(y1, theta) @ z1
+
+  x3 = axis_angle_to_rot(x2, phi) @ x2
+  y3 = axis_angle_to_rot(x2, phi) @ y2
+
+  x0 = np.array([1.0, 0, 0])
+  y0 = np.array([0, 1.0, 0])
+  z0 = np.array([0, 0, 1.0])
+
+  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
+  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
+
+  y2 = axis_angle_to_rot(z0, psi_out) @ y0
+  z2 = axis_angle_to_rot(y2, theta_out) @ z0
+
+  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
+
+  return np.array([phi_out, theta_out, psi_out])
+
+
+def ned_euler_from_ecef_single(ecef_init, ecef_pose):
+  """
+  Convert ECEF Euler angles (roll, pitch, yaw) at a given ECEF origin
+  to equivalent NED Euler angles.
+  """
+  converter = LocalCoord(ecef=ecef_init)
+
+  x0 = np.array([1.0, 0, 0])
+  y0 = np.array([0, 1.0, 0])
+  z0 = np.array([0, 0, 1.0])
+
+  phi, theta, psi = ecef_pose
+
+  x1 = axis_angle_to_rot(z0, psi) @ x0
+  y1 = axis_angle_to_rot(z0, psi) @ y0
+  z1 = axis_angle_to_rot(z0, psi) @ z0
+
+  x2 = axis_angle_to_rot(y1, theta) @ x1
+  y2 = axis_angle_to_rot(y1, theta) @ y1
+  z2 = axis_angle_to_rot(y1, theta) @ z1
+
+  x3 = axis_angle_to_rot(x2, phi) @ x2
+  y3 = axis_angle_to_rot(x2, phi) @ y2
+
+  zero = np.array(ecef_init)
+  x0 = converter.ned2ecef_single([1, 0, 0]) - zero
+  y0 = converter.ned2ecef_single([0, 1, 0]) - zero
+  z0 = converter.ned2ecef_single([0, 0, 1]) - zero
+
+  psi_out = np.arctan2(np.dot(x3, y0), np.dot(x3, x0))
+  theta_out = np.arctan2(-np.dot(x3, z0), np.sqrt(np.dot(x3, x0)**2 + np.dot(x3, y0)**2))
+
+  y2 = axis_angle_to_rot(z0, psi_out) @ y0
+  z2 = axis_angle_to_rot(y2, theta_out) @ z0
+
+  phi_out = np.arctan2(np.dot(y3, z2), np.dot(y3, y2))
+
+  return np.array([phi_out, theta_out, psi_out])

openpilot/common/util.cc

@@ -0,0 +1,317 @@
+#include "common/util.h"
+#include "common/swaglog.h"
+
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/resource.h>
+
+#include <cassert>
+#include <cerrno>
+#include <cstring>
+#include <dirent.h>
+#include <fstream>
+#include <iomanip>
+#include <random>
+#include <sstream>
+#include <limits>
+
+#ifdef __linux__
+#include <sys/prctl.h>
+#include <sys/syscall.h>
+#ifndef __USE_GNU
+#define __USE_GNU
+#endif
+#include <sched.h>
+#endif  // __linux__
+
+namespace util {
+
+void set_thread_name(const char* name) {
+#ifdef __linux__
+  // pthread_setname_np is dumb (fails instead of truncates)
+  prctl(PR_SET_NAME, (unsigned long)name, 0, 0, 0);
+#endif
+}
+
+int set_realtime_priority(int level) {
+#ifdef __linux__
+  long tid = syscall(SYS_gettid);
+
+  // should match python using chrt
+  struct sched_param sa;
+  memset(&sa, 0, sizeof(sa));
+  sa.sched_priority = level;
+  return sched_setscheduler(tid, SCHED_FIFO, &sa);
+#else
+  return -1;
+#endif
+}
+
+int set_core_affinity(std::vector<int> cores) {
+#ifdef __linux__
+  long tid = syscall(SYS_gettid);
+  cpu_set_t cpu;
+
+  CPU_ZERO(&cpu);
+  for (const int n : cores) {
+    CPU_SET(n, &cpu);
+  }
+  return sched_setaffinity(tid, sizeof(cpu), &cpu);
+#else
+  return -1;
+#endif
+}
+
+int set_file_descriptor_limit(uint64_t limit_val) {
+  struct rlimit limit;
+  int status;
+
+  if ((status = getrlimit(RLIMIT_NOFILE, &limit)) < 0)
+    return status;
+
+  limit.rlim_cur = limit_val;
+  if ((status = setrlimit(RLIMIT_NOFILE, &limit)) < 0)
+    return status;
+
+  return 0;
+}
+
+std::string read_file(const std::string& fn) {
+  std::ifstream f(fn, std::ios::binary | std::ios::in);
+  if (f.is_open()) {
+    f.seekg(0, std::ios::end);
+    std::streamsize size = f.tellg();
+    // seekg and tellg on a directory doesn't return pos_type(-1) but max(streamsize)
+    if (f.good() && size > 0 && size < std::numeric_limits<std::streamsize>::max()) {
+      std::string result(size, '\0');
+      f.seekg(0, std::ios::beg);
+      f.read(result.data(), size);
+      // return either good() or has reached end-of-file (e.g. /sys/power/wakeup_count)
+      if (f.good() || f.eof()) {
+        result.resize(f.gcount());
+        return result;
+      }
+    }
+    // fallback for files created on read, e.g. procfs
+    std::stringstream buffer;
+    buffer << f.rdbuf();
+    return buffer.str();
+  }
+  return std::string();
+}
+
+std::map<std::string, std::string> read_files_in_dir(const std::string &path) {
+  std::map<std::string, std::string> ret;
+  DIR *d = opendir(path.c_str());
+  if (!d) return ret;
+
+  struct dirent *de = NULL;
+  while ((de = readdir(d))) {
+    if (de->d_type != DT_DIR) {
+      ret[de->d_name] = util::read_file(path + "/" + de->d_name);
+    }
+  }
+
+  closedir(d);
+  return ret;
+}
+
+int write_file(const char* path, const void* data, size_t size, int flags, mode_t mode) {
+  int fd = HANDLE_EINTR(open(path, flags, mode));
+  if (fd == -1) {
+    return -1;
+  }
+  ssize_t n = HANDLE_EINTR(write(fd, data, size));
+  close(fd);
+  return (n >= 0 && (size_t)n == size) ? 0 : -1;
+}
+
+FILE* safe_fopen(const char* filename, const char* mode) {
+  FILE* fp = NULL;
+  do {
+    fp = fopen(filename, mode);
+  } while ((nullptr == fp) && (errno == EINTR));
+  return fp;
+}
+
+size_t safe_fwrite(const void* ptr, size_t size, size_t count, FILE* stream) {
+  size_t written = 0;
+  do {
+    size_t ret = ::fwrite((void*)((char *)ptr + written * size), size, count - written, stream);
+    if (ret == 0 && errno != EINTR) break;
+    written += ret;
+  } while (written != count);
+  return written;
+}
+
+int safe_fflush(FILE *stream) {
+  int ret = EOF;
+  do {
+    ret = fflush(stream);
+  } while ((EOF == ret) && (errno == EINTR));
+  return ret;
+}
+
+int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_msg) {
+  int ret;
+  do {
+    ret = ioctl(fd, request, argp);
+  } while ((ret == -1) && (errno == EINTR));
+
+  if (ret == -1 && exception_msg) {
+    LOGE("safe_ioctl error: %s %s(%d) (fd: %d request: %lx argp: %p)", exception_msg, strerror(errno), errno, fd, request, argp);
+    throw std::runtime_error(exception_msg);
+  }
+  return ret;
+}
+
+std::string readlink(const std::string &path) {
+  char buff[4096];
+  ssize_t len = ::readlink(path.c_str(), buff, sizeof(buff)-1);
+  if (len != -1) {
+    buff[len] = '\0';
+    return std::string(buff);
+  }
+  return "";
+}
+
+bool file_exists(const std::string& fn) {
+  struct stat st = {};
+  return stat(fn.c_str(), &st) != -1;
+}
+
+static bool createDirectory(std::string dir, mode_t mode) {
+  auto verify_dir = [](const std::string& path) -> bool {
+    struct stat st = {};
+    return (stat(path.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR);
+  };
+  // remove trailing /'s
+  while (dir.size() > 1 && dir.back() == '/') {
+    dir.pop_back();
+  }
+  // try to mkdir this directory
+  if (mkdir(dir.c_str(), mode) == 0) return true;
+  if (errno == EEXIST) return verify_dir(dir);
+  if (errno != ENOENT) return false;
+
+  // mkdir failed because the parent dir doesn't exist, so try to create it
+  size_t slash = dir.rfind('/');
+  if ((slash == std::string::npos || slash < 1) ||
+      !createDirectory(dir.substr(0, slash), mode)) {
+    return false;
+  }
+
+  // try again
+  if (mkdir(dir.c_str(), mode) == 0) return true;
+  return errno == EEXIST && verify_dir(dir);
+}
+
+bool create_directories(const std::string& dir, mode_t mode) {
+  if (dir.empty()) return false;
+  return createDirectory(dir, mode);
+}
+
+std::string getenv(const char* key, std::string default_val) {
+  const char* val = ::getenv(key);
+  return val ? val : default_val;
+}
+
+int getenv(const char* key, int default_val) {
+  const char* val = ::getenv(key);
+  return val ? atoi(val) : default_val;
+}
+
+float getenv(const char* key, float default_val) {
+  const char* val = ::getenv(key);
+  return val ? atof(val) : default_val;
+}
+
+std::string hexdump(const uint8_t* in, const size_t size) {
+  std::stringstream ss;
+  ss << std::hex << std::setfill('0');
+  for (size_t i = 0; i < size; i++) {
+    ss << std::setw(2) << static_cast<unsigned int>(in[i]);
+  }
+  return ss.str();
+}
+
+int random_int(int min, int max) {
+  std::random_device dev;
+  std::mt19937 rng(dev());
+  std::uniform_int_distribution<std::mt19937::result_type> dist(min, max);
+  return dist(rng);
+}
+
+std::string random_string(std::string::size_type length) {
+  const std::string chrs = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+  std::mt19937 rg{std::random_device{}()};
+  std::uniform_int_distribution<std::string::size_type> pick(0, chrs.length() - 1);
+  std::string s;
+  s.reserve(length);
+  while (length--) {
+    s += chrs[pick(rg)];
+  }
+  return s;
+}
+
+bool starts_with(const std::string &s1, const std::string &s2) {
+  return strncmp(s1.c_str(), s2.c_str(), s2.size()) == 0;
+}
+
+bool ends_with(const std::string& s, const std::string& suffix) {
+  return s.size() >= suffix.size() &&
+         strcmp(s.c_str() + (s.size() - suffix.size()), suffix.c_str()) == 0;
+}
+
+std::string strip(const std::string &str) {
+  auto should_trim = [](unsigned char ch) {
+    return std::isspace(ch) || ch == '\0';
+  };
+
+  size_t start = 0;
+  while (start < str.size() && should_trim(static_cast<unsigned char>(str[start]))) {
+    start++;
+  }
+
+  if (start == str.size()) {
+    return "";
+  }
+
+  size_t end = str.size() - 1;
+  while (end > 0 && should_trim(static_cast<unsigned char>(str[end]))) {
+    end--;
+  }
+
+  return str.substr(start, end - start + 1);
+}
+
+std::string check_output(const std::string& command) {
+  char buffer[128];
+  std::string result;
+  std::unique_ptr<FILE, int(*)(FILE*)> pipe(popen(command.c_str(), "r"), pclose);
+
+  if (!pipe) {
+    return "";
+  }
+
+  while (fgets(buffer, std::size(buffer), pipe.get()) != nullptr) {
+    result += std::string(buffer);
+  }
+
+  return result;
+}
+
+bool system_time_valid() {
+  // Default to August 26, 2024
+  tm min_tm = {.tm_mday = 26, .tm_mon = 7, .tm_year = 2024 - 1900};
+  time_t min_date = mktime(&min_tm);
+
+  struct stat st;
+  if (stat("/lib/systemd/systemd", &st) == 0) {
+    min_date = std::max(min_date, st.st_mtime + 86400);  // Add 1 day (86400 seconds)
+  }
+
+  return time(nullptr) > min_date;
+}
+
+}  // namespace util

openpilot/common/util.h

@@ -0,0 +1,195 @@
+#pragma once
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <csignal>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+// keep trying if x gets interrupted by a signal
+#define HANDLE_EINTR(x)                                        \
+  ({                                                           \
+    decltype(x) ret_;                                          \
+    int try_cnt = 0;                                           \
+    do {                                                       \
+      ret_ = (x);                                              \
+    } while (ret_ == -1 && errno == EINTR && try_cnt++ < 100); \
+    ret_;                                                       \
+  })
+
+#ifndef sighandler_t
+typedef void (*sighandler_t)(int sig);
+#endif
+
+const double MILE_TO_KM = 1.609344;
+const double KM_TO_MILE = 1. / MILE_TO_KM;
+const double MS_TO_KPH = 3.6;
+const double MS_TO_MPH = MS_TO_KPH * KM_TO_MILE;
+const double METER_TO_MILE = KM_TO_MILE / 1000.0;
+const double METER_TO_FOOT = 3.28084;
+const double METER_TO_KM = 1. / 1000.0;
+
+#define ALIGNED_SIZE(x, align) (((x) + (align)-1) & ~((align)-1))
+
+namespace util {
+
+void set_thread_name(const char* name);
+int set_realtime_priority(int level);
+int set_core_affinity(std::vector<int> cores);
+int set_file_descriptor_limit(uint64_t limit);
+
+// ***** math helpers *****
+
+// map x from [a1, a2] to [b1, b2]
+template <typename T>
+T map_val(T x, T a1, T a2, T b1, T b2) {
+  x = std::clamp(x, a1, a2);
+  T ra = a2 - a1;
+  T rb = b2 - b1;
+  return (x - a1) * rb / ra + b1;
+}
+
+// ***** string helpers *****
+
+template <typename... Args>
+std::string string_format(const std::string& format, Args... args) {
+  size_t size = snprintf(nullptr, 0, format.c_str(), args...) + 1;
+  std::unique_ptr<char[]> buf(new char[size]);
+  snprintf(buf.get(), size, format.c_str(), args...);
+  return std::string(buf.get(), buf.get() + size - 1);
+}
+
+std::string getenv(const char* key, std::string default_val = "");
+int getenv(const char* key, int default_val);
+float getenv(const char* key, float default_val);
+
+std::string hexdump(const uint8_t* in, const size_t size);
+bool starts_with(const std::string &s1, const std::string &s2);
+bool ends_with(const std::string &s, const std::string &suffix);
+std::string strip(const std::string &str);
+
+// ***** random helpers *****
+int random_int(int min, int max);
+std::string random_string(std::string::size_type length);
+
+// **** file helpers *****
+std::string read_file(const std::string& fn);
+std::map<std::string, std::string> read_files_in_dir(const std::string& path);
+int write_file(const char* path, const void* data, size_t size, int flags = O_WRONLY, mode_t mode = 0664);
+
+FILE* safe_fopen(const char* filename, const char* mode);
+size_t safe_fwrite(const void * ptr, size_t size, size_t count, FILE * stream);
+int safe_fflush(FILE *stream);
+int safe_ioctl(int fd, unsigned long request, void *argp, const char* exception_msg = nullptr);
+
+std::string readlink(const std::string& path);
+bool file_exists(const std::string& fn);
+bool create_directories(const std::string &dir, mode_t mode);
+
+std::string check_output(const std::string& command);
+
+inline void check_system(const std::string& cmd) {
+  int ret = std::system(cmd.c_str());
+  if (ret != 0) {
+    fprintf(stderr, "system command failed (%d): %s\n", ret, cmd.c_str());
+  }
+}
+
+bool system_time_valid();
+
+inline void sleep_for(const int milliseconds) {
+  if (milliseconds > 0) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+  }
+}
+
+}  // namespace util
+
+class ExitHandler {
+public:
+  ExitHandler() {
+    std::signal(SIGINT, (sighandler_t)set_do_exit);
+    std::signal(SIGTERM, (sighandler_t)set_do_exit);
+
+#ifndef __APPLE__
+    std::signal(SIGPWR, (sighandler_t)set_do_exit);
+#endif
+  }
+  inline static std::atomic<bool> power_failure = false;
+  inline static std::atomic<int> signal = 0;
+  inline operator bool() { return do_exit; }
+  inline ExitHandler& operator=(bool v) {
+    signal = 0;
+    do_exit = v;
+    return *this;
+  }
+private:
+  static void set_do_exit(int sig) {
+#ifndef __APPLE__
+    power_failure = (sig == SIGPWR);
+#endif
+    signal = sig;
+    do_exit = true;
+  }
+  inline static std::atomic<bool> do_exit = false;
+};
+
+struct unique_fd {
+  unique_fd(int fd = -1) : fd_(fd) {}
+  unique_fd& operator=(unique_fd&& uf) {
+    fd_ = uf.fd_;
+    uf.fd_ = -1;
+    return *this;
+  }
+  ~unique_fd() {
+    if (fd_ != -1) close(fd_);
+  }
+  operator int() const { return fd_; }
+  int fd_;
+};
+
+class FirstOrderFilter {
+public:
+  FirstOrderFilter(float x0, float ts, float dt, bool initialized = true) {
+    k_ = (dt / ts) / (1.0 + dt / ts);
+    x_ = x0;
+    initialized_ = initialized;
+  }
+  inline float update(float x) {
+    if (initialized_) {
+      x_ = (1. - k_) * x_ + k_ * x;
+    } else {
+      initialized_ = true;
+      x_ = x;
+    }
+    return x_;
+  }
+  inline void reset(float x) { x_ = x; }
+  inline float x(){ return x_; }
+
+private:
+  float x_, k_;
+  bool initialized_;
+};
+
+template<typename T>
+void update_max_atomic(std::atomic<T>& max, T const& value) {
+  T prev = max;
+  while (prev < value && !max.compare_exchange_weak(prev, value)) {}
+}
+
+typedef struct Rect {
+  int x;
+  int y;
+  int w;
+  int h;
+} Rect;

openpilot/common/utils.py

@@ -0,0 +1,272 @@
+import io
+import os
+import tempfile
+import contextlib
+import subprocess
+import time
+import functools
+from subprocess import Popen, PIPE, TimeoutExpired
+import zstandard as zstd
+
+LOG_COMPRESSION_LEVEL = 10  # little benefit up to level 15. level ~17 is a small step change
+
+class Timer:
+  """Simple lap timer for profiling sequential operations."""
+
+  def __init__(self):
+    self._start = self._lap = time.monotonic()
+    self._sections = {}
+
+  def lap(self, name):
+    now = time.monotonic()
+    self._sections[name] = now - self._lap
+    self._lap = now
+
+  @property
+  def total(self):
+    return time.monotonic() - self._start
+
+  def fmt(self, duration):
+    parts = ", ".join(f"{k}={v:.2f}s" + (f" ({duration/v:.0f}x)" if k == 'render' and v > 0 else "") for k, v in self._sections.items())
+    total = self.total
+    realtime = f"{duration/total:.1f}x realtime" if total > 0 else "N/A"
+    return f"{duration}s in {total:.1f}s ({realtime}) | {parts}"
+
+def sudo_write(val: str, path: str) -> None:
+  try:
+    with open(path, 'w') as f:
+      f.write(str(val))
+  except PermissionError:
+    subprocess.run(f"sudo chmod a+w {path}", shell=True)
+    try:
+      with open(path, 'w') as f:
+        f.write(str(val))
+    except PermissionError:
+      # fallback for debugfs files
+      subprocess.run(f"sudo su -c 'echo {val} > {path}'", shell=True)
+
+
+def sudo_read(path: str) -> str:
+  try:
+    return subprocess.check_output(["sudo", "cat", "--", path], encoding='utf8').strip()
+  except Exception:
+    return ""
+
+
+class MovingAverage:
+  def __init__(self, window_size: int):
+    self.window_size: int = window_size
+    self.buffer: list[float] = [0.0] * window_size
+    self.index: int = 0
+    self.count: int = 0
+    self.sum: float = 0.0
+
+  def add_value(self, new_value: float):
+    # Update the sum: subtract the value being replaced and add the new value
+    self.sum -= self.buffer[self.index]
+    self.buffer[self.index] = new_value
+    self.sum += new_value
+
+    # Update the index in a circular manner
+    self.index = (self.index + 1) % self.window_size
+
+    # Track the number of added values (for partial windows)
+    self.count = min(self.count + 1, self.window_size)
+
+  def get_average(self) -> float:
+    if self.count == 0:
+      return float('nan')
+    return self.sum / self.count
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None,
+                 overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)
+
+
+def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
+  if not should_compress:
+    file_size = os.path.getsize(filepath)
+    file_stream = open(filepath, "rb")
+    return file_stream, file_size
+
+  # Compress the file on the fly
+  compressed_stream = io.BytesIO()
+  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
+
+  with open(filepath, "rb") as f:
+    compressor.copy_stream(f, compressed_stream)
+    compressed_size = compressed_stream.tell()
+    compressed_stream.seek(0)
+    return compressed_stream, compressed_size
+
+
+# remove all keys that end in DEPRECATED, plus any "deprecated" group
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED') or k == 'deprecated':
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+
+
+@contextlib.contextmanager
+def managed_proc(cmd: list[str], env: dict[str, str]):
+  proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE)
+  try:
+    yield proc
+  finally:
+    if proc.poll() is None:
+      proc.terminate()
+    try:
+      proc.wait(timeout=5)
+    except TimeoutExpired:
+      proc.kill()
+
+
+def tabulate(tabular_data, headers=(), tablefmt="simple", floatfmt="g", stralign="left", numalign=None):
+  rows = [list(row) for row in tabular_data]
+
+  def fmt(val):
+    if isinstance(val, str):
+      return val
+    if isinstance(val, (bool, int)):
+      return str(val)
+    try:
+      return format(val, floatfmt)
+    except (TypeError, ValueError):
+      return str(val)
+
+  formatted = [[fmt(c) for c in row] for row in rows]
+  hdrs = [str(h) for h in headers] if headers else None
+
+  ncols = max((len(r) for r in formatted), default=0)
+  if hdrs:
+    ncols = max(ncols, len(hdrs))
+  if ncols == 0:
+    return ""
+
+  for r in formatted:
+    r.extend([""] * (ncols - len(r)))
+  if hdrs:
+    hdrs.extend([""] * (ncols - len(hdrs)))
+
+  widths = [0] * ncols
+  if hdrs:
+    for i in range(ncols):
+      widths[i] = len(hdrs[i])
+  for row in formatted:
+    for i in range(ncols):
+      widths[i] = max(widths[i], max(len(ln) for ln in row[i].split('\n')))
+
+  def _align(s, w):
+    if stralign == "center":
+      return s.center(w)
+    return s.ljust(w)
+
+  if tablefmt == "html":
+    parts = ["<table>"]
+    if hdrs:
+      parts.append("<thead>")
+      parts.append("<tr>" + "".join(f"<th>{h}</th>" for h in hdrs) + "</tr>")
+      parts.append("</thead>")
+    parts.append("<tbody>")
+    for row in formatted:
+      parts.append("<tr>" + "".join(f"<td>{c}</td>" for c in row) + "</tr>")
+    parts.append("</tbody>")
+    parts.append("</table>")
+    return "\n".join(parts)
+
+  if tablefmt == "simple_grid":
+    def _sep(left, mid, right):
+      return left + mid.join("\u2500" * (w + 2) for w in widths) + right
+
+    top, mid_sep, bot = _sep("\u250c", "\u252c", "\u2510"), _sep("\u251c", "\u253c", "\u2524"), _sep("\u2514", "\u2534", "\u2518")
+
+    def _fmt_row(cells):
+      split = [c.split('\n') for c in cells]
+      nlines = max(len(s) for s in split)
+      for s in split:
+        s.extend([""] * (nlines - len(s)))
+      return ["\u2502" + "\u2502".join(f" {_align(split[i][li], widths[i])} " for i in range(ncols)) + "\u2502" for li in range(nlines)]
+
+    lines = [top]
+    if hdrs:
+      lines.extend(_fmt_row(hdrs))
+      lines.append(mid_sep)
+    for ri, row in enumerate(formatted):
+      lines.extend(_fmt_row(row))
+      lines.append(mid_sep if ri < len(formatted) - 1 else bot)
+    return "\n".join(lines)
+
+  # simple
+  gap = "  "
+  lines = []
+  if hdrs:
+    lines.append(gap.join(h.ljust(w) for h, w in zip(hdrs, widths, strict=True)))
+    lines.append(gap.join("-" * w for w in widths))
+  for row in formatted:
+    lines.append(gap.join(_align(row[i], widths[i]) for i in range(ncols)))
+  return "\n".join(lines)
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          print(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        print(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator

openpilot/common/version.h

@@ -0,0 +1 @@
+#define COMMA_VERSION "0.11.2"

openpilot/common/version.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+from dataclasses import dataclass
+from functools import cache
+import json
+import os
+import pathlib
+import subprocess
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.swaglog import cloudlog
+from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date
+
+RELEASE_SP_BRANCHES = ['release-c3', 'release', 'release-tizi', 'release-tici', 'release-tizi-staging', 'release-tici-staging']
+TESTED_SP_BRANCHES = ['staging-c3', 'staging-c3-new', 'staging']
+MASTER_SP_BRANCHES = ['master']
+RELEASE_BRANCHES = ['release-tizi-staging', 'release-mici-staging', 'release-tizi', 'release-mici', 'nightly']
+TESTED_BRANCHES = RELEASE_BRANCHES + ['devel-staging', 'nightly-dev'] + RELEASE_SP_BRANCHES + TESTED_SP_BRANCHES
+
+SP_BRANCH_MIGRATIONS = {
+  ("tici", "staging-c3-new"): "staging-tici",
+  ("tici", "dev-c3-new"): "staging-tici",
+  ("tici", "master"): "master-tici",
+  ("tici", "master-dev-c3-new"): "master-tici",
+  ("tizi", "staging-c3-new"): "staging",
+  ("tizi", "dev-c3-new"): "dev",
+  ("tizi", "master-dev-c3-new"): "master-dev",
+  ("tizi", "release3"): "release-tizi",
+  ("tizi", "release3-staging"): "release-tizi-staging",
+  ("mici", "release3"): "release-mici",
+  ("mici", "release3-staging"): "release-mici-staging",
+}
+
+BUILD_METADATA_FILENAME = "build.json"
+
+training_version: str = "0.2.0"
+terms_version: str = "2"
+terms_version_sp: str = "1.0"
+sunnylink_consent_version: str = "1.0"
+sunnylink_consent_declined: str = "-1"
+
+
+def get_version(path: str = BASEDIR) -> str:
+  with open(os.path.join(path, "openpilot", "sunnypilot", "common", "version.h")) as _versionf:
+    version = _versionf.read().split('"')[1]
+  return version
+
+
+def get_release_notes(path: str = BASEDIR) -> str:
+  with open(os.path.join(path, "CHANGELOG.md")) as f:
+    return f.read().split('\n\n', 1)[0]
+
+
+@cache
+def is_prebuilt(path: str = BASEDIR) -> bool:
+  return os.path.exists(os.path.join(path, 'prebuilt'))
+
+
+@cache
+def is_dirty(cwd: str = BASEDIR) -> bool:
+  if not get_origin() or not get_short_branch():
+    return True
+
+  dirty = False
+  try:
+    # Actually check dirty files
+    if not is_prebuilt(cwd):
+      # This is needed otherwise touched files might show up as modified
+      try:
+        subprocess.check_call(["git", "update-index", "--refresh"], cwd=cwd)
+      except subprocess.CalledProcessError:
+        pass
+
+      branch = get_branch()
+      if not branch:
+        return True
+      dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"], cwd=cwd)) != 0
+  except subprocess.CalledProcessError:
+    cloudlog.exception("git subprocess failed while checking dirty")
+    dirty = True
+
+  return dirty
+
+
+@dataclass
+class OpenpilotMetadata:
+  version: str
+  release_notes: str
+  git_commit: str
+  git_origin: str
+  git_commit_date: str
+  build_style: str
+  is_dirty: bool  # whether there are local changes
+
+  @property
+  def short_version(self) -> str:
+    return self.version.split('-')[0]
+
+  @property
+  def comma_remote(self) -> bool:
+    # note to fork maintainers, this is used for release metrics. please do not
+    # touch this to get rid of the orange startup alert. there's better ways to do that
+    return self.git_normalized_origin == "github.com/commaai/openpilot"
+
+  @property
+  def sunnypilot_remote(self) -> bool:
+    return self.git_normalized_origin in ("github.com/sunnypilot/sunnypilot",
+                                          "github.com/sunnypilot/openpilot",
+                                          "github.com/sunnyhaibin/sunnypilot",
+                                          "github.com/sunnyhaibin/openpilot")
+
+  @property
+  def git_normalized_origin(self) -> str:
+    return self.git_origin \
+      .replace("git@", "", 1) \
+      .replace(".git", "", 1) \
+      .replace("https://", "", 1) \
+      .replace(":", "/", 1)
+
+
+@dataclass
+class BuildMetadata:
+  channel: str
+  openpilot: OpenpilotMetadata
+
+  @property
+  def tested_channel(self) -> bool:
+    return self.channel in TESTED_BRANCHES
+
+  @property
+  def release_channel(self) -> bool:
+    return self.channel in RELEASE_BRANCHES
+
+  @property
+  def release_sp_channel(self) -> bool:
+    return self.channel in RELEASE_SP_BRANCHES
+
+  @property
+  def canonical(self) -> str:
+    return f"{self.openpilot.version}-{self.openpilot.git_commit}-{self.openpilot.build_style}"
+
+  @property
+  def ui_description(self) -> str:
+    return f"{self.openpilot.version} / {self.openpilot.git_commit[:6]} / {self.channel}"
+
+  @property
+  def master_channel(self) -> bool:
+    return self.channel in MASTER_SP_BRANCHES
+
+  @property
+  def development_channel(self) -> bool:
+    return self.channel == "dev" or self.channel.startswith("dev-") or self.channel.endswith("-prebuilt")
+
+  @property
+  def channel_type(self) -> str:
+    if self.channel.endswith("-tici"):
+      return "tici"
+    elif self.development_channel:
+      return "development"
+    elif self.tested_channel:
+      return "staging"
+    elif self.master_channel:
+      return "master"
+    elif self.release_channel or self.release_sp_channel:
+      return "release"
+    else:
+      return "feature"
+
+
+def build_metadata_from_dict(build_metadata: dict) -> BuildMetadata:
+  channel = build_metadata.get("channel", "unknown")
+  openpilot_metadata = build_metadata.get("openpilot", {})
+  version = openpilot_metadata.get("version", "unknown")
+  release_notes = openpilot_metadata.get("release_notes", "unknown")
+  git_commit = openpilot_metadata.get("git_commit", "unknown")
+  git_origin = openpilot_metadata.get("git_origin", "unknown")
+  git_commit_date = openpilot_metadata.get("git_commit_date", "unknown")
+  build_style = openpilot_metadata.get("build_style", "unknown")
+  return BuildMetadata(channel,
+            OpenpilotMetadata(
+              version=version,
+              release_notes=release_notes,
+              git_commit=git_commit,
+              git_origin=git_origin,
+              git_commit_date=git_commit_date,
+              build_style=build_style,
+              is_dirty=False))
+
+
+def get_build_metadata(path: str = BASEDIR) -> BuildMetadata:
+  build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME
+
+  if build_metadata_path.exists():
+    build_metadata = json.loads(build_metadata_path.read_text())
+    return build_metadata_from_dict(build_metadata)
+
+  git_folder = pathlib.Path(path) / ".git"
+
+  if git_folder.exists():
+    return BuildMetadata(get_short_branch(path),
+                    OpenpilotMetadata(
+                      version=get_version(path),
+                      release_notes=get_release_notes(path),
+                      git_commit=get_commit(path),
+                      git_origin=get_origin(path),
+                      git_commit_date=get_commit_date(path),
+                      build_style="unknown",
+                      is_dirty=is_dirty(path)))
+
+  cloudlog.exception("unable to get build metadata")
+  raise Exception("invalid build metadata")
+
+
+if __name__ == "__main__":
+  print(get_build_metadata())

openpilot/common/yuv.cc

@@ -0,0 +1,132 @@
+#include "common/yuv.h"
+
+#include <algorithm>
+#include <cstring>
+
+namespace yuv {
+
+namespace {
+
+inline uint8_t clamp_u8(int v) {
+  return static_cast<uint8_t>(std::clamp(v, 0, 255));
+}
+
+void copy_plane(const uint8_t *src, int src_stride,
+                uint8_t *dst, int dst_stride,
+                int width, int height) {
+  if (src_stride == width && dst_stride == width) {
+    std::memcpy(dst, src, static_cast<size_t>(width) * height);
+    return;
+  }
+  for (int y = 0; y < height; ++y) {
+    std::memcpy(dst + y * dst_stride, src + y * src_stride, width);
+  }
+}
+
+void scale_plane_point(const uint8_t *src, int src_stride, int src_width, int src_height,
+                       uint8_t *dst, int dst_stride, int dst_width, int dst_height) {
+  if (src_width == dst_width && src_height == dst_height) {
+    copy_plane(src, src_stride, dst, dst_stride, dst_width, dst_height);
+    return;
+  }
+  for (int y = 0; y < dst_height; ++y) {
+    const int sy = y * src_height / dst_height;
+    const uint8_t *src_row = src + sy * src_stride;
+    uint8_t *dst_row = dst + y * dst_stride;
+    for (int x = 0; x < dst_width; ++x) {
+      dst_row[x] = src_row[x * src_width / dst_width];
+    }
+  }
+}
+
+// BT.601 limited range → RGB (integer form used widely, incl. similar to libyuv).
+inline void yuv_to_rgb(int y, int u, int v, uint8_t *r, uint8_t *g, uint8_t *b) {
+  const int c = (y - 16) * 298;
+  const int d = u - 128;
+  const int e = v - 128;
+  *r = clamp_u8((c + 409 * e + 128) >> 8);
+  *g = clamp_u8((c - 100 * d - 208 * e + 128) >> 8);
+  *b = clamp_u8((c + 516 * d + 128) >> 8);
+}
+
+}  // namespace
+
+void nv12_to_i420(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_uv, int src_stride_uv,
+                  uint8_t *dst_y, int dst_stride_y,
+                  uint8_t *dst_u, int dst_stride_u,
+                  uint8_t *dst_v, int dst_stride_v,
+                  int width, int height) {
+  copy_plane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+
+  const int uv_width = width / 2;
+  const int uv_height = height / 2;
+  for (int y = 0; y < uv_height; ++y) {
+    const uint8_t *uv = src_uv + y * src_stride_uv;
+    uint8_t *u = dst_u + y * dst_stride_u;
+    uint8_t *v = dst_v + y * dst_stride_v;
+    for (int x = 0; x < uv_width; ++x) {
+      u[x] = uv[2 * x];
+      v[x] = uv[2 * x + 1];
+    }
+  }
+}
+
+void i420_to_nv12(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_u, int src_stride_u,
+                  const uint8_t *src_v, int src_stride_v,
+                  uint8_t *dst_y, int dst_stride_y,
+                  uint8_t *dst_uv, int dst_stride_uv,
+                  int width, int height) {
+  copy_plane(src_y, src_stride_y, dst_y, dst_stride_y, width, height);
+
+  const int uv_width = width / 2;
+  const int uv_height = height / 2;
+  for (int y = 0; y < uv_height; ++y) {
+    const uint8_t *u = src_u + y * src_stride_u;
+    const uint8_t *v = src_v + y * src_stride_v;
+    uint8_t *uv = dst_uv + y * dst_stride_uv;
+    for (int x = 0; x < uv_width; ++x) {
+      uv[2 * x] = u[x];
+      uv[2 * x + 1] = v[x];
+    }
+  }
+}
+
+void i420_scale(const uint8_t *src_y, int src_stride_y,
+                const uint8_t *src_u, int src_stride_u,
+                const uint8_t *src_v, int src_stride_v,
+                int src_width, int src_height,
+                uint8_t *dst_y, int dst_stride_y,
+                uint8_t *dst_u, int dst_stride_u,
+                uint8_t *dst_v, int dst_stride_v,
+                int dst_width, int dst_height) {
+  scale_plane_point(src_y, src_stride_y, src_width, src_height,
+                    dst_y, dst_stride_y, dst_width, dst_height);
+  scale_plane_point(src_u, src_stride_u, src_width / 2, src_height / 2,
+                    dst_u, dst_stride_u, dst_width / 2, dst_height / 2);
+  scale_plane_point(src_v, src_stride_v, src_width / 2, src_height / 2,
+                    dst_v, dst_stride_v, dst_width / 2, dst_height / 2);
+}
+
+void nv12_to_rgba(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_uv, int src_stride_uv,
+                  uint8_t *dst_rgba, int dst_stride_rgba,
+                  int width, int height) {
+  for (int y = 0; y < height; ++y) {
+    const uint8_t *y_row = src_y + y * src_stride_y;
+    const uint8_t *uv_row = src_uv + (y / 2) * src_stride_uv;
+    uint8_t *dst = dst_rgba + y * dst_stride_rgba;
+    for (int x = 0; x < width; ++x) {
+      const int uv_x = (x & ~1);
+      uint8_t r, g, b;
+      yuv_to_rgb(y_row[x], uv_row[uv_x], uv_row[uv_x + 1], &r, &g, &b);
+      dst[4 * x + 0] = r;
+      dst[4 * x + 1] = g;
+      dst[4 * x + 2] = b;
+      dst[4 * x + 3] = 255;
+    }
+  }
+}
+
+}  // namespace yuv

openpilot/common/yuv.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <cstdint>
+
+// NV12: Y plane + interleaved UV. I420: planar Y, U, V.
+
+namespace yuv {
+
+// Deinterleave NV12 UV into planar I420.
+void nv12_to_i420(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_uv, int src_stride_uv,
+                  uint8_t *dst_y, int dst_stride_y,
+                  uint8_t *dst_u, int dst_stride_u,
+                  uint8_t *dst_v, int dst_stride_v,
+                  int width, int height);
+
+// Interleave planar I420 UV into NV12.
+void i420_to_nv12(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_u, int src_stride_u,
+                  const uint8_t *src_v, int src_stride_v,
+                  uint8_t *dst_y, int dst_stride_y,
+                  uint8_t *dst_uv, int dst_stride_uv,
+                  int width, int height);
+
+// Point-sample scale I420 (equivalent to libyuv::I420Scale + kFilterNone).
+void i420_scale(const uint8_t *src_y, int src_stride_y,
+                const uint8_t *src_u, int src_stride_u,
+                const uint8_t *src_v, int src_stride_v,
+                int src_width, int src_height,
+                uint8_t *dst_y, int dst_stride_y,
+                uint8_t *dst_u, int dst_stride_u,
+                uint8_t *dst_v, int dst_stride_v,
+                int dst_width, int dst_height);
+
+// Convert NV12 to packed RGBA (R,G,B,A bytes — suitable for GL_RGBA).
+// BT.601 limited-range, matching common libyuv defaults.
+void nv12_to_rgba(const uint8_t *src_y, int src_stride_y,
+                  const uint8_t *src_uv, int src_stride_uv,
+                  uint8_t *dst_rgba, int dst_stride_rgba,
+                  int width, int height);
+
+}